[ { "sites": [ "classifieds" ], "task_id": 0, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} on this site.", "intent": "Find me the cheapest blue kayak on this site.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "blue kayak" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=4799", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 0 }, { "sites": [ "classifieds" ], "task_id": 1, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}}. It should be between {{range}}.", "intent": "Find me the cheapest red Toyota. It should be between $3000 to $6000.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "red Toyota", "range": "$3000 to $6000" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=35838", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 1 }, { "sites": [ "classifieds" ], "task_id": 2, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}}. It should be between {{range}}.", "intent": "Find me the most recently posted item in the \"Jewelry\" category with a red gem. It should be between $300 to $600.", "image": null, "instantiation_dict": { "attribute": "most recently posted", "item": "item in the \"Jewelry\" category with a red gem", "range": "$300 to $600" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=10066", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 1 }, { "sites": [ "classifieds" ], "task_id": 3, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}}. It should be between {{range}}.", "intent": "Find me the most recently posted item in the \"Photo + video\" category that is not completely black in color. It should be between $1000 to $1200.", "image": null, "instantiation_dict": { "attribute": "most recently posted", "item": "item in the \"Photo + video\" category that is not completely black in color", "range": "$1000 to $1200" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=39343", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 1 }, { "sites": [ "classifieds" ], "task_id": 4, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing of the white car and change the price to $25000. Update the price in the description as well.", "image": null, "instantiation_dict": { "item": "of the white car", "action": "change the price to $25000. Update the price in the description as well" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84144", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "25000.00" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84144", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_include": [ "$25000 |OR| $25,000" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84144", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_exclude": [ "30000", "30,000" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 5, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing of the white car and delete it.", "image": null, "instantiation_dict": { "item": "of the white car", "action": "delete it" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84144", "locator": "", "required_contents": { "must_include": [ "404" ] } } ] }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 6, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Return the links of the {{attribute}} {{items}}.", "intent": "Return the links of the 3 most recent motorcycles within $1000 to $2000 that are not orange.", "image": null, "instantiation_dict": { "attribute": "3 most recent", "items": "motorcycles within $1000 to $2000 that are not orange" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__CLASSIFIEDS__/index.php?page=item&id=4208", "__CLASSIFIEDS__/index.php?page=item&id=65357", "__CLASSIFIEDS__/index.php?page=item&id=11549" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 3 }, { "sites": [ "classifieds" ], "task_id": 7, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Return the links of the {{attribute}} {{items}}.", "intent": "Return the links of the 2 most recent items in the \"Cell phones\" category within $300 to $600 that are white in color.", "image": null, "instantiation_dict": { "attribute": "2 most recent", "items": "items in the \"Cell phones\" category within $300 to $600 that are white in color" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__CLASSIFIEDS__/index.php?page=item&id=40404", "__CLASSIFIEDS__/index.php?page=item&id=38125" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 3 }, { "sites": [ "classifieds" ], "task_id": 8, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Help me make a post selling this item and navigate to it. Price it at $10 cheaper than the most similar item on the site {{range}} (of the same color). If there are multiple similar items, use the most recent posting.", "intent": "Help me make a post selling this item and navigate to it. Price it at $10 cheaper than the most similar item on the site between $250-300 (of the same color). If there are multiple similar items, use the most recent posting.", "image": "https://images.pexels.com/photos/6869207/pexels-photo-6869207.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "range": "between $250-300" }, "require_reset": true, "eval": { "eval_types": [ "program_html", "page_image_query" ], "reference_answers": null, "program_html": [ { "url": "last", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "270.00" ] } } ], "page_image_query": [ { "eval_image_url": "last", "eval_image_class": ".item-photos", "eval_fuzzy_image_match": "https://images.pexels.com/photos/6869207/pexels-photo-6869207.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2" } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "Closest item: __CLASSIFIEDS__/index.php?page=item&id=81838 or __CLASSIFIEDS__/index.php?page=item&id=32109", "intent_template_id": 4 }, { "sites": [ "classifieds" ], "task_id": 9, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Help me make a post selling this item and navigate to it. Price it at $10 cheaper than the most similar item on the site {{range}} (of the same color). If there are multiple similar items, use the most recent posting.", "intent": "Help me make a post selling this item and navigate to it. Price it at $10 cheaper than the most similar item on the site between $750-800 (of the same color). If there are multiple similar items, use the most recent posting.", "image": "__CLASSIFIEDS__/oc-content/uploads/42906/42906.png", "instantiation_dict": { "range": "between $750-800" }, "require_reset": true, "eval": { "eval_types": [ "program_html", "page_image_query" ], "reference_answers": null, "program_html": [ { "url": "last", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "785.00" ] } } ], "page_image_query": [ { "eval_image_url": "last", "eval_image_class": ".item-photos", "eval_fuzzy_image_match": "__CLASSIFIEDS__/oc-content/uploads/42906/42906.png" } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "Closest item: __CLASSIFIEDS__/index.php?page=item&id=42906", "intent_template_id": 4 }, { "sites": [ "classifieds" ], "task_id": 10, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "What is the {{attribute}} of {{item}}?", "intent": "What is the seat height in inches of the smaller piece of furniture on this page?", "image": null, "instantiation_dict": { "attribute": "seat height in inches", "item": "the smaller piece of furniture on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "21" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=43887", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 5 }, { "sites": [ "classifieds" ], "task_id": 11, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=7", "geolocation": null, "intent_template": "What is the {{attribute}} of {{item}}?", "intent": "What is the size of the wheels in inches of the first blue bike on this page?", "image": null, "instantiation_dict": { "attribute": "size of the wheels in inches", "item": "the first blue bike on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "26" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66613", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 5 }, { "sites": [ "classifieds" ], "task_id": 12, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "What is the {{attribute}} of {{item}}?", "intent": "What is the color of most recently listed motorcycle?", "image": null, "instantiation_dict": { "attribute": "color", "item": "most recently listed motorcycle" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "red" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=9068", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 5 }, { "sites": [ "classifieds" ], "task_id": 13, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "What is the {{attribute}} of {{item}}?", "intent": "What is the color of the most expensive item in the \"Boats\" category?", "image": null, "instantiation_dict": { "attribute": "color", "item": "the most expensive item in the \"Boats\" category" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "white" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=40932", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 5 }, { "sites": [ "classifieds" ], "task_id": 14, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=4&iPage=2&sShowAs=gallery", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is the email of the seller of {{item}}?", "intent": "What is the email of the seller of the painting in the second row?", "image": null, "instantiation_dict": { "item": "the painting in the second row" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "olga.jones341@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 6 }, { "sites": [ "classifieds" ], "task_id": 15, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=21", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is the email of the seller of {{item}} on this page?", "intent": "What is the email of the seller of the guitar in the red case on this page?", "image": null, "instantiation_dict": { "item": "the guitar in the red case" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "haruto_abdullah497@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "__CLASSIFIEDS__/index.php?page=item&id=67527", "intent_template_id": 7 }, { "sites": [ "classifieds" ], "task_id": 16, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=3", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is the email of the seller of {{item}} on this page?", "intent": "What is the email of the seller of the item with the coffee mug in the picture on this page?", "image": null, "instantiation_dict": { "item": "the item with the coffee mug in the picture" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "layla_garcia352@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=15810", "intent_template_id": 7 }, { "sites": [ "classifieds" ], "task_id": 17, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "Show me the {{item}}.", "intent": "Show me the cheapest bike with red handlebars between $900-950.", "image": null, "instantiation_dict": { "item": "cheapest bike with red handlebars between $900-950" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=79747 |OR| __CLASSIFIEDS__/index.php?page=item&id=11376", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 8 }, { "sites": [ "classifieds" ], "task_id": 18, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "Show me the {{item}}.", "intent": "Show me the most recently posted blue iPhone.", "image": null, "instantiation_dict": { "item": "most recently posted blue iPhone" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=38173", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 8 }, { "sites": [ "classifieds" ], "task_id": 19, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "Show me the {{item}}.", "intent": "Show me the most recently posted painting in the \"Arts + crafts\" category between $80-100 that shows the sea.", "image": null, "instantiation_dict": { "item": "most recently posted painting in the \"Arts + crafts\" category between $80-100 that shows the sea" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=47487", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 8 }, { "sites": [ "classifieds" ], "task_id": 20, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "Show me the {{item}}.", "intent": "Show me the most recently posted white Xbox.", "image": null, "instantiation_dict": { "item": "most recently posted white Xbox" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=49061", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 8 }, { "sites": [ "classifieds" ], "task_id": 21, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "Show me the {{item}}.", "intent": "Show me the newest \"RVs + campers\" listing that is in a dark color.", "image": null, "instantiation_dict": { "item": "newest \"RVs + campers\" listing that is in a dark color" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=33164", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 8 }, { "sites": [ "classifieds" ], "task_id": 22, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&sShowAs=gallery", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "How {{quantity}} does the {{item}} have?", "intent": "How many miles does the red car in the second row have?", "image": null, "instantiation_dict": { "quantity": "many miles", "item": "red car in the second row" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "103K |OR| 103,000 |OR| 103000" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 9 }, { "sites": [ "classifieds" ], "task_id": 23, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&sShowAs=gallery", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "How {{quantity}} does the {{item}} have?", "intent": "How many miles does the second car in the third row have?", "image": null, "instantiation_dict": { "quantity": "many miles", "item": "second car in the third row" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "64K |OR| 64,000 |OR| 64000" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 9 }, { "sites": [ "classifieds" ], "task_id": 24, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&sShowAs=gallery&iPage=5", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "How {{quantity}} does the {{item}} have?", "intent": "How many miles does the black truck on this page have?", "image": null, "instantiation_dict": { "quantity": "many miles", "item": "black truck on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "Miles not listed for this car." }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "__CLASSIFIEDS__/index.php?page=item&id=83979 does not list miles", "intent_template_id": 9 }, { "sites": [ "classifieds" ], "task_id": 25, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "How many {{item}} {{criteria}}?", "intent": "How many red boats were posted on 16th November 2023?", "image": null, "instantiation_dict": { "item": "red boats", "criteria": "were posted on 16th November 2023" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "1 |OR| one" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=49894", "intent_template_id": 10 }, { "sites": [ "classifieds" ], "task_id": 26, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "How many {{item}} {{criteria}}?", "intent": "How many yellow or blue motorcycles in total were posted on 25th October 2023?", "image": null, "instantiation_dict": { "item": "yellow or blue motorcycles in total", "criteria": "were posted on 25th October 2023" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "2 |OR| two" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=74997 and __CLASSIFIEDS__/index.php?page=item&id=58261", "intent_template_id": 10 }, { "sites": [ "classifieds" ], "task_id": 27, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "How many {{item}} {{criteria}}?", "intent": "How many of the listings in \"Rvs + campers\" posted on 16th November 2023 show the interior of the car?", "image": null, "instantiation_dict": { "item": "of the listings in \"Rvs + campers\" posted on 16th November 2023", "criteria": "show the interior of the car" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "3 |OR| three" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=32934 and __CLASSIFIEDS__/index.php?page=item&id=74707 and __CLASSIFIEDS__/index.php?page=item&id=32672", "intent_template_id": 10 }, { "sites": [ "classifieds" ], "task_id": 28, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Add a comment on the {{item}} with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "Add a comment on the most recently posted painting of a bridge with the title \"Question\" and text \"When did you acquire it?\".", "image": null, "instantiation_dict": { "item": "most recently posted painting of a bridge", "title": "Question", "comment": "When did you acquire it?" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=12085", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Question by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=12085", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "When did you acquire it?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=12085", "intent_template_id": 11 }, { "sites": [ "classifieds" ], "task_id": 29, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Add a comment on the {{item}} with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "Add a comment on the most recently posted red car with the title \"Nice car\" and text \"Is it still available?\".", "image": null, "instantiation_dict": { "item": "most recently posted red car", "title": "Nice car", "comment": "Is it still available?" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=66091", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Nice car by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=66091", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Is it still available?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=66091", "intent_template_id": 11 }, { "sites": [ "classifieds" ], "task_id": 30, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Add a comment on the {{item}} with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "Add a comment on the most expensive black couch with the title \"Interesting Couch\" and text \"Is the price negotiable?\".", "image": null, "instantiation_dict": { "item": "most expensive black couch", "title": "Interesting Couch", "comment": "Is the price negotiable?" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=44542", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interesting Couch by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=44542", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Is the price negotiable?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=44542", "intent_template_id": 11 }, { "sites": [ "classifieds" ], "task_id": 31, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find {{item}} and post a comment offering $10 less than their asking price.", "intent": "Find the latest listing of a white Google Pixel phone and post a comment offering $10 less than their asking price.", "image": null, "instantiation_dict": { "item": "the latest listing of a white Google Pixel phone" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=47273", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "$250" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=47273", "intent_template_id": 12 }, { "sites": [ "classifieds" ], "task_id": 32, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find {{item}} and post a comment offering $10 less than their asking price.", "intent": "Find this exact bike that's listed for $300-500 and post a comment offering $10 less than their asking price.", "image": "__CLASSIFIEDS__/oc-content/uploads/9689/9689.png", "instantiation_dict": { "item": "this exact bike that's listed for $300-500" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=9689", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "$385" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=9689", "intent_template_id": 12 }, { "sites": [ "classifieds" ], "task_id": 33, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find {{item}} and post a comment offering $10 less than their asking price.", "intent": "Find the latest listing of a white fridge and post a comment offering $10 less than their asking price.", "image": null, "instantiation_dict": { "item": "the latest listing of a white fridge" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=62633", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "$115" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=62633", "intent_template_id": 12 }, { "sites": [ "classifieds" ], "task_id": 34, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\"{{detail}} and return the links of {{target}}.", "intent": "Search for \"painting\" and return the links of the most recent three items that feature animals.", "image": null, "instantiation_dict": { "phrase": "painting", "detail": "", "target": "the most recent three items that feature animals" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__CLASSIFIEDS__/index.php?page=item&id=14761", "__CLASSIFIEDS__/index.php?page=item&id=32385", "__CLASSIFIEDS__/index.php?page=item&id=47173" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 13 }, { "sites": [ "classifieds" ], "task_id": 35, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\"{{detail}} and return the links of {{target}}.", "intent": "Search for \"loveseat\" in the Furniture category and return the links of the most recent two items that are not black or gray.", "image": null, "instantiation_dict": { "phrase": "loveseat", "detail": " in the Furniture category", "category": "Furniture", "target": "the most recent two items that are not black or gray" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__CLASSIFIEDS__/index.php?page=item&id=33441", "__CLASSIFIEDS__/index.php?page=item&id=42263" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 13 }, { "sites": [ "classifieds", "shopping" ], "task_id": 36, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__", "geolocation": null, "intent_template": "Find all listings for this exact item on OSClass and compare its price to the cheapest listing on OneStopMarket. Return the link of {{target}}.", "intent": "Find all listings for this exact item on OSClass and compare its price to the cheapest listing on OneStopMarket. Return the link of the cheaper item.", "image": "__SHOPPING__/media/catalog/product/cache/829a59e57f886f8cf0598ffca4f8a940/B/0/B07SK4W1VJ.0.jpg", "instantiation_dict": { "target": "the cheaper item" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__SHOPPING__/luigi-s-mansion-3-nintendo-switch.html" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 14 }, { "sites": [ "classifieds", "shopping" ], "task_id": 37, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__", "geolocation": null, "intent_template": "Find all listings for this exact item on OSClass and compare its price to the cheapest listing on OneStopMarket. Return the link of {{target}}.", "intent": "Find all listings for this exact item on OSClass and compare its price to the cheapest listing on OneStopMarket. Return the link of the cheapest item on both sites.", "image": "__SHOPPING__/media/catalog/product/cache/829a59e57f886f8cf0598ffca4f8a940/B/0/B07YZQ9QMD.0.jpg", "instantiation_dict": { "target": "the cheapest item on both sites" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "__SHOPPING__/nintendo-switch-w-neon-blue-neon-red-joy-con-mario-kart-8-deluxe-full-game-download-switch.html", "__CLASSIFIEDS__/index.php?page=item&id=59435" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 14 }, { "sites": [ "classifieds" ], "task_id": 38, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the three most recent {{item}} in the \"{{category}}\" category, and tell me {{attribute}}.", "intent": "Find the three most recent grey or silver sedans in the \"Cars + trucks\" category, and tell me how much mileage each has.", "image": null, "instantiation_dict": { "item": "grey or silver sedans", "category": "Cars + trucks", "attribute": "how much mileage each has" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "175k |OR| 175000 |OR| 175,000", "153k |OR| 153000 |OR| 153,000", "154k |OR| 154000 |OR| 154,000" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=60945 and __CLASSIFIEDS__/index.php?page=item&id=52305 and __CLASSIFIEDS__/index.php?page=item&id=50777", "intent_template_id": 15 }, { "sites": [ "classifieds" ], "task_id": 39, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the three most recent {{item}} in the \"{{category}}\" category, and tell me {{attribute}}.", "intent": "Find the three most recent bikes with predominantly red frames in the \"Bikes\" category, and tell me the size of their frames in inches.", "image": null, "instantiation_dict": { "item": "bikes with predominantly red frames", "category": "Bikes", "attribute": "the size of their frames in inches" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "18", "14", "19" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=70291 and __CLASSIFIEDS__/index.php?page=item&id=76502 and __CLASSIFIEDS__/index.php?page=item&id=66548", "intent_template_id": 15 }, { "sites": [ "classifieds" ], "task_id": 40, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and tell me {{detail}}.", "intent": "Search for \"dishwasher\" and tell me the brand of the most recent listing of a stainless steel one.", "image": null, "instantiation_dict": { "phrase": "dishwasher", "detail": "the brand of the most recent listing of a stainless steel one" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "LG" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=18607", "intent_template_id": 16 }, { "sites": [ "classifieds" ], "task_id": 41, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=8&sShowAs=gallery", "geolocation": null, "intent_template": "What is the price range of {{items}}?", "intent": "What is the price range of the listings in the second row of this page?", "image": null, "instantiation_dict": { "items": "the listings in the second row of this page" }, "viewport_size": { "width": 1280 }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "1200", "23750" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 17 }, { "sites": [ "classifieds" ], "task_id": 42, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=12&sShowAs=gallery", "geolocation": null, "intent_template": "What is the price range of {{items}}?", "intent": "What is the price range of the listings in the last two rows of this page?", "image": null, "instantiation_dict": { "items": "the listings in the last two rows of this page" }, "viewport_size": { "width": 1280 }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "5", "120" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 17 }, { "sites": [ "classifieds" ], "task_id": 43, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&sShowAs=gallery&iPage=4", "geolocation": null, "intent_template": "What is the price range of {{items}}?", "intent": "What is the price range of all the red vehicles on this page?", "image": null, "instantiation_dict": { "items": "all the red vehicles on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "7800", "9999" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 17 }, { "sites": [ "classifieds" ], "task_id": 44, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it.", "image": "__CLASSIFIEDS__/oc-content/uploads/26492/26492.png", "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=26492", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 18 }, { "sites": [ "classifieds" ], "task_id": 45, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it.", "image": "__CLASSIFIEDS__/oc-content/uploads/45196/45196.png", "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=45196", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 18 }, { "sites": [ "classifieds" ], "task_id": 46, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it.", "image": "__CLASSIFIEDS__/oc-content/uploads/66067/66067.png", "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66067", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 18 }, { "sites": [ "classifieds" ], "task_id": 47, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall that it was very expensive.", "image": "__CLASSIFIEDS__/oc-content/uploads/7434/7434.png", "instantiation_dict": { "hint": "that it was very expensive" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=7434", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 19 }, { "sites": [ "classifieds" ], "task_id": 48, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall seeing it in either the Collectibles or Antiques section.", "image": "__CLASSIFIEDS__/oc-content/uploads/20629/20629.png", "instantiation_dict": { "hint": "seeing it in either the Collectibles or Antiques section" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=20629", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 19 }, { "sites": [ "classifieds" ], "task_id": 49, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=14", "geolocation": null, "intent_template": "How {{quantity}} does the {{item}} have?", "intent": "How much RAM (in GB) does the item with blue LED lights on this page have?", "image": null, "instantiation_dict": { "quantity": "much RAM (in GB)", "item": "item with blue LED lights on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "64 |OR| 64GB" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "__CLASSIFIEDS__/index.php?page=item&id=60306", "intent_template_id": 9 }, { "sites": [ "classifieds" ], "task_id": 50, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=6", "geolocation": null, "intent_template": "What is the email of the seller of {{item}}?", "intent": "What is the email of the seller of the red palette on this page?", "image": null, "instantiation_dict": { "item": "the red palette on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "sofia_kumar@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "__CLASSIFIEDS__/index.php?page=item&id=65673", "intent_template_id": 6 }, { "sites": [ "classifieds" ], "task_id": 51, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the most recent {{type}} in the \"{{category}}\" category of {{item}}.", "intent": "Find me the most recent painting in the \"Arts + crafts\" category of a town that looks like this.", "image": "https://images.pexels.com/photos/161815/santorini-oia-greece-water-161815.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "type": "painting", "category": "Arts + crafts", "item": "a town that looks like this" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=39336", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "6th page of the Arts + crafts section", "intent_template_id": 20 }, { "sites": [ "classifieds" ], "task_id": 52, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the most recent {{type}} in the \"{{category}}\" category of {{item}}.", "intent": "Find me the most recent painting in the \"Arts + crafts\" category of something that looks close to this.", "image": "https://images.pexels.com/photos/1330219/pexels-photo-1330219.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "type": "painting", "category": "Arts + crafts", "item": "something that looks close to this" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=60133", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "2nd page of the Arts + crafts section", "intent_template_id": 20 }, { "sites": [ "classifieds" ], "task_id": 53, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and find the {{product}}.", "intent": "Search for \"gnome\" and find the item that would make the best gift for someone who goes to this school.", "image": "__WIKIPEDIA__/wikipedia_en_all_maxi_2022-05/I/Pitt_Panthers_wordmark.svg.png.webp", "instantiation_dict": { "phrase": "gnome", "product": "item that would make the best gift for someone who goes to this school" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=19511", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 21 }, { "sites": [ "classifieds" ], "task_id": 54, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the most recently listed item in the \"{{category}}\" category that {{condition}}.", "intent": "Find the most recently listed item in the \"Video gaming\" category that is related to this game I used to play.", "image": "__WIKIPEDIA__/wikipedia_en_all_maxi_2022-05/I/Donkey_Kong_NES_Screenshot.png.webp", "instantiation_dict": { "category": "Video gaming", "condition": "is related to this game I used to play" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=787", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 22 }, { "sites": [ "classifieds" ], "task_id": 55, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the most recently listed item in the \"{{category}}\" category that {{condition}}.", "intent": "Find the most recently listed item in the \"Collectibles\" category that depicts the inventors of this.", "image": "__WIKIPEDIA__/wikipedia_en_all_maxi_2022-05/I/First_flight2.jpg.webp", "instantiation_dict": { "category": "Collectibles", "condition": "depicts the inventors of this" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=78765", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "Inventors are the Wright brothers", "intent_template_id": 22 }, { "sites": [ "classifieds" ], "task_id": 56, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} that {{condition}}.", "intent": "Find the cheapest snowblower that is not red in color.", "image": null, "instantiation_dict": { "item": "cheapest snowblower", "condition": "is not red in color" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=25929", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 23 }, { "sites": [ "classifieds" ], "task_id": 57, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Add a comment on the {{item}} with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "Add a comment on the two most recently posted blue bikes between $100-150 with the title \"Interested in your bike\" and text \"Is it still available?\".", "image": null, "instantiation_dict": { "item": "two most recently posted blue bikes between $100-150", "title": "Interested in your bike", "comment": "Is it still available?" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=66613", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interested in your bike" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=66613", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Is it still available?" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=71313", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interested in your bike" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=71313", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Is it still available?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=66613 and __CLASSIFIEDS__/index.php?page=item&id=71313", "intent_template_id": 11 }, { "sites": [ "classifieds" ], "task_id": 58, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category of {{location}} and find me {{item}}.", "intent": "Explore the \"Furniture\" category of Washington, D.C. and find me the most recent blue chair.", "image": null, "instantiation_dict": { "category": "Furniture", "location": "Washington, D.C.", "item": "the most recent blue chair" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=37760", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 24 }, { "sites": [ "classifieds" ], "task_id": 59, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} with {{detail}} from the image {{location}}.", "intent": "Find the cheapest video game item with the animal from the image on its display.", "image": "https://images.pexels.com/photos/86405/penguin-funny-blue-water-86405.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "cheapest video game item", "detail": "the animal", "location": "on its display" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6247", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 25 }, { "sites": [ "classifieds" ], "task_id": 60, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the most expensive video game item where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/620335/pexels-photo-620335.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive video game item", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=4188", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 61, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the most expensive video game item where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/344034/pexels-photo-344034.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive video game item", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=42756", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 62, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} with {{detail}} from the image {{location}}.", "intent": "Find the most expensive video game item with the character from the image on its display.", "image": "https://images.pexels.com/photos/2854693/pexels-photo-2854693.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive video game item", "detail": "the character", "location": "on its display" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=19137", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 25 }, { "sites": [ "classifieds" ], "task_id": 63, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} with {{detail}} from the image {{location}}.", "intent": "Find the most expensive video game item with the character from the image on its display.", "image": "https://images.pexels.com/photos/7768661/pexels-photo-7768661.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive video game item", "detail": "the character", "location": "on its display" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=17379", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 25 }, { "sites": [ "classifieds" ], "task_id": 64, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the video game item that costs exactly $500 where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/620335/pexels-photo-620335.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "video game item that costs exactly $500", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=73629", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 65, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the cheapest video game item where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/274422/pexels-photo-274422.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "cheapest video game item", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=28239", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 66, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the cheapest video game item where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/718952/pexels-photo-718952.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "cheapest video game item", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6175", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 67, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the cheapest video game item where I can roleplay the situation in the image.", "image": "https://images.pexels.com/photos/2346/sport-high-united-states-of-america-ball.jpg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "cheapest video game item", "detail": "I can roleplay the situation in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=7114", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 68, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} where {{detail}}.", "intent": "Find the cheapest Tiger Woods video game item where the person on the cover has a red shirt.", "image": null, "instantiation_dict": { "product": "cheapest Tiger Woods video game item", "detail": "the person on the cover has a red shirt" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=2278", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 26 }, { "sites": [ "classifieds" ], "task_id": 69, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} which {{detail}}.", "intent": "Find the most expensive item posted from Delaware which is related to the brand on the hat in the image.", "image": "https://images.pexels.com/photos/16976200/pexels-photo-16976200/free-photo-of-girl-sitting-on-a-curb.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive item posted from Delaware", "detail": "is related to the brand on the hat in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=15047", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 27 }, { "sites": [ "classifieds" ], "task_id": 70, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} from {{location}} {{condition}}.", "intent": "Find the most expensive tractor from Delaware that is green.", "image": null, "instantiation_dict": { "item": "most expensive tractor", "location": "Delaware", "condition": "that is green" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=68860", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 28 }, { "sites": [ "classifieds" ], "task_id": 71, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} which {{detail}}.", "intent": "Find the most expensive item posted from Delaware which is of the same item as in the image.", "image": "https://images.pexels.com/photos/359995/pexels-photo-359995.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "product": "most expensive item posted from Delaware", "detail": "is of the same item as in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=12399 |OR| __CLASSIFIEDS__/index.php?page=item&id=53168", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 27 }, { "sites": [ "classifieds" ], "task_id": 72, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I like {{attribute}}. Can you find the {{detail}} where I can buy similar items, if there's one?", "intent": "I like the products in the image. Can you find the cheapest posting from Delaware where I can buy similar items, if there's one?", "image": "https://images.pexels.com/photos/5883534/pexels-photo-5883534.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "attribute": "the products in the image", "detail": "cheapest posting from Delaware" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=43255", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 29 }, { "sites": [ "classifieds" ], "task_id": 73, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me {{product}} that {{condition}} in the image.", "intent": "Find me the cheapest posting from Delaware that has the same brand as the product in the image.", "image": "https://images.pexels.com/photos/1858115/pexels-photo-1858115.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1", "instantiation_dict": { "condition": "has the same brand as the product", "product": "the cheapest posting from Delaware" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=78325", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 30 }, { "sites": [ "classifieds" ], "task_id": 74, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} from {{location}} {{condition}}.", "intent": "Find the most expensive truck from Ohio that is white.", "image": null, "instantiation_dict": { "item": "most expensive truck", "location": "Ohio", "condition": "that is white" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=14818", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 28 }, { "sites": [ "classifieds" ], "task_id": 75, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing of the white vase and change the price to $120 (including in the description).", "image": null, "instantiation_dict": { "item": "of the white vase", "action": "change the price to $120 (including in the description)" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84148", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "120.00" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84148", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_include": [ "$120" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84148", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_exclude": [ "$80" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 76, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing of the blue bike and change the price to $85.50 (including in the description).", "image": null, "instantiation_dict": { "item": "of the blue bike", "action": "change the price to $85.50 (including in the description)" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84146", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "85.50" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84146", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_include": [ "$85.50" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84146", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_exclude": [ "$250" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 77, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing of the all-black bike and change the price (including in the description) to one dollar more than the cheapest bike on the site (ignoring bikes that cost $1).", "image": null, "instantiation_dict": { "item": "of the all-black bike", "action": "change the price (including in the description) to one dollar more than the cheapest bike on the site (ignoring bikes that cost $1)" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84145", "locator": "func:get_query_text(__page__, '.price')", "required_contents": { "must_include": [ "6.00" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84145", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_include": [ "$6" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=84145", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_exclude": [ "$300" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 78, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sOrder=dt_pub_date&iOrderType=desc&iPage=4", "geolocation": null, "intent_template": "Find me {{product}} that {{condition}} in the image.", "intent": "Find me an item on this page that references the city in the image.", "image": "https://images.pexels.com/photos/12493794/pexels-photo-12493794.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "an item on this page", "condition": "references the city" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=14342", "program_html": [], "url_note": "EXACT" }, "intent_template_id": 30, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "" }, { "sites": [ "classifieds" ], "task_id": 79, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sOrder=dt_pub_date&iOrderType=desc&iPage=4", "geolocation": null, "intent_template": "Find me {{product}} that {{condition}} in the image.", "intent": "Find me an item on this page that references the person in the image.", "image": "https://images.pexels.com/photos/1604849/pexels-photo-1604849.jpeg", "instantiation_dict": { "product": "an item on this page", "condition": "references the person" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=44608", "program_html": [], "url_note": "EXACT" }, "intent_template_id": 30, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "" }, { "sites": [ "classifieds" ], "task_id": 80, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&iPage=4&sOrder=i_price&iOrderType=asc", "geolocation": null, "intent_template": "Tell me the {{detail}} that {{product}}?", "intent": "Tell me the first names of the listers who posted the items on the page that do not include a book in its image?", "image": null, "instantiation_dict": { "detail": "first names of the listers who posted the items on the page", "product": "do not include a book in its image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Yuna", "Siti" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=33101 and __CLASSIFIEDS__/index.php?page=item&id=45618", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 31 }, { "sites": [ "classifieds" ], "task_id": 81, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest book with a hurricane on the cover.", "image": null, "instantiation_dict": { "item": "book", "condition": "with a hurricane on the cover" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=4727", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 82, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the most expensive purple hard-case book, and tell me how much it costs in dollars.", "image": null, "instantiation_dict": { "product": "most expensive purple hard-case book" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "1100" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=44383", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 83, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=19&sOrder=i_price&iOrderType=desc&iPage=2", "geolocation": null, "intent_template": "Tell me the {{detail}} that {{product}}?", "intent": "Tell me the last name of the lister who posted the item on the page that does not include jewelry in its image?", "image": null, "instantiation_dict": { "detail": "last name of the lister who posted the item on the page", "product": "does not include jewelry in its image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Nowak" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6407", "program_html": [], "string_note": "", "reference_answer_raw_annotation": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 31 }, { "sites": [ "classifieds" ], "task_id": 84, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the most expensive ring that has an image taken as a selfie, and tell me how much it costs in dollars.", "image": null, "instantiation_dict": { "product": "most expensive ring that has an image taken as a selfie" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "6000" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=43966", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 85, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the most expensive ring that does not show the physical ring in the image, and tell me how much it costs in dollars.", "image": null, "instantiation_dict": { "product": "most expensive ring that does not show the physical ring in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "3200" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=57020", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 86, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest Toyota that is the same color as the cheaper Toyota in my listings.", "image": null, "instantiation_dict": { "item": "Toyota", "condition": "that is the same color as the cheaper Toyota in my listings" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=72103", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 87, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=15&sOrder=i_price&iOrderType=desc&iPage=5", "geolocation": null, "intent_template": "Find the {{item}} on this page {{condition}}.", "intent": "Find the most similar item on this page as shown in this image.", "image": "https://images.pexels.com/photos/3051966/pexels-photo-3051966.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "most similar item", "condition": "as shown in this image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=34463", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 34 }, { "sites": [ "classifieds" ], "task_id": 88, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} from {{location}} {{condition}}.", "intent": "Find the most expensive red vehicle in the \"Cars + trucks\" category from Virginia with trees in the background of its image.", "image": null, "instantiation_dict": { "item": "most expensive red vehicle in the \"Cars + trucks\" category", "location": "Virginia", "condition": "with trees in the background of its image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=50736", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 28 }, { "sites": [ "classifieds" ], "task_id": 89, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} from {{location}} {{condition}}.", "intent": "Find the most expensive car from Virginia that is neon green.", "image": null, "instantiation_dict": { "item": "most expensive car", "location": "Virginia", "condition": "that is neon green" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=63326", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 28 }, { "sites": [ "classifieds" ], "task_id": 90, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sRegion=9254928&sOrder=i_price&iOrderType=asc&sCategory=10&iPage=2", "geolocation": null, "intent_template": "Find the {{item}} on this page {{condition}}.", "intent": "Find the car on this page that has the most discoloring.", "image": null, "instantiation_dict": { "item": "car", "condition": "that has the most discoloring" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=52649", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 34 }, { "sites": [ "classifieds" ], "task_id": 91, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest car listing in the \"Cars + trucks\" category from Ohio that has an image taken from inside the car.", "image": null, "instantiation_dict": { "item": "car listing in the \"Cars + trucks\" category from Ohio", "condition": "that has an image taken from inside the car" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=81402", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 92, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} from {{location}} {{condition}}.", "intent": "Find the most expensive TV from Maryland that displays an ongoing NFL game.", "image": null, "instantiation_dict": { "item": "most expensive TV", "location": "Maryland", "condition": "that displays an ongoing NFL game" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=23785", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 28 }, { "sites": [ "classifieds" ], "task_id": 93, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sRegion=7361885&sCategory=15&sOrder=i_price&iOrderType=asc&iPage=4", "geolocation": null, "intent_template": "Find the {{product}} which {{detail}}.", "intent": "Find the electronics item on the page which is compatible with this image.", "image": "https://images.pexels.com/photos/1334597/pexels-photo-1334597.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "electronics item on the page", "detail": "is compatible with this image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=42923", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 27 }, { "sites": [ "classifieds" ], "task_id": 94, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=16", "geolocation": null, "intent_template": "Find the {{item}} on this page {{condition}}.", "intent": "Find the animal on this page that has someone riding it in the image.", "image": null, "instantiation_dict": { "item": "animal", "condition": "that has someone riding it in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=79622", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 34 }, { "sites": [ "classifieds" ], "task_id": 95, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} with {{attribute}}. Add a {{rating}} rating with title \"{{title}}\" and text \"{{text}}\".", "intent": "Find the most expensive boat with a listing image showing it on water. Add a 5 star rating with title \"Awesome boat\" and text \"Mine is similar!\".", "image": null, "instantiation_dict": { "item": "most expensive boat", "attribute": "a listing image showing it on water", "rating": "5 star", "title": "Awesome boat", "text": "Mine is similar!" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=40932", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Awesome boat by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=40932", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Mine is similar!", "5 of 5" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 35 }, { "sites": [ "classifieds" ], "task_id": 96, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=3&sOrder=i_price&iOrderType=asc&iPage=12", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the most similar item on this page as given in the image, and tell me how much it costs in dollars.", "image": "https://images.pexels.com/photos/4107286/pexels-photo-4107286.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most similar item on this page as given in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "14" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=5939", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 97, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Households\" category, and find the most recent item designed in the shape of an animal.", "image": null, "instantiation_dict": { "category": "Households", "item": "most recent item designed in the shape of an animal" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=57906", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 98, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "How many {{attribute}} are {{listing detail}}?", "intent": "How many hours are on the engine of the most recently listed red boat?", "image": null, "instantiation_dict": { "attribute": "hours", "listing detail": "on the engine of the most recently listed red boat" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "80 |OR| eighty" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 37 }, { "sites": [ "classifieds" ], "task_id": 99, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the most expensive wheelchair lift that has multiple people in the image of the listing, and tell me how much it costs in dollars.", "image": null, "instantiation_dict": { "product": "most expensive wheelchair lift that has multiple people in the image of the listing" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "2995" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=9830", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 100, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "What is {{attribute}} on the {{product}}?", "intent": "What is the number shown on the image of the most expensive race car in the Arts + crafts section?", "image": null, "instantiation_dict": { "attribute": "the number shown", "product": "image of the most expensive race car in the Arts + crafts section" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "61 |OR| sixty one" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=2354", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 38 }, { "sites": [ "classifieds" ], "task_id": 101, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}} in this image.", "intent": "Explore the \"Art + crafts\" category, and find the most expensive painting of the city in this image.", "image": "https://images.pexels.com/photos/12493795/pexels-photo-12493795.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "category": "Art + crafts", "item": "most expensive painting of the city" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=83533", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 39 }, { "sites": [ "classifieds" ], "task_id": 102, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the height in inches of the most recently listed item in Art + crafts section portraying the person in the image.", "image": "https://images.pexels.com/photos/767276/pexels-photo-767276.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "height in inches of the most recently listed item in Art + crafts section portraying the person in the image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "8 1/2 |OR| eight and a half |OR| 8.5 |OR| eight point five" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 103, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}} in this image.", "intent": "Explore the \"Arts + crafts\" category, and find the most recently listed item portraying the person in this image.", "image": "https://images.pexels.com/photos/37072/abraham-lincoln-lincoln-memorial-washington-dc-lincoln.jpg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "category": "Arts + crafts", "item": "most recently listed item portraying the person" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=77577", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 39 }, { "sites": [ "classifieds" ], "task_id": 104, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} with {{attribute}}. Add a {{rating}} rating with title \"{{title}}\" and text \"{{text}}\".", "intent": "Find the most recently listed RV with an iPhone screenshot as its image. Add a 3 star rating with title \"Any other pics?\" and text \"Do you have a higher quality picture?\".", "image": null, "instantiation_dict": { "item": "most recently listed RV", "attribute": "an iPhone screenshot as its image", "rating": "3 star", "title": "Any other pics?", "text": "Do you have a higher quality picture?" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=21206", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Any other pics? by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=21206", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Do you have a higher quality picture?", "3 of 5" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 35 }, { "sites": [ "classifieds" ], "task_id": 105, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest guitar with a red velvet lined case.", "image": null, "instantiation_dict": { "item": "guitar", "condition": "with a red velvet lined case" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=23225", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 106, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "What is the email of the seller of {{item}}?", "intent": "What is the email of the seller of the most expensive item in Photo + videos that has an animal for its image?", "image": null, "instantiation_dict": { "item": "the most expensive item in Photo + videos that has an animal for its image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "miguel_ito@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 6 }, { "sites": [ "classifieds" ], "task_id": 107, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the color of the rims on the most expensive bike in West Virginia.", "image": null, "instantiation_dict": { "product": "color of the rims on the most expensive bike in West Virginia" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "purple", "blue |OR| cyan" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=52440", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 108, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the name of the lister with the most expensive green vehicle from West Virginia.", "image": null, "instantiation_dict": { "product": "name of the lister with the most expensive green vehicle from West Virginia" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Daiki Lee" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=7438", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 109, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "How {{quantity}} does the {{item}} have?", "intent": "How many miles does the most expensive white sports car in the \"Cars + trucks\" category from Ohio have?", "image": null, "instantiation_dict": { "quantity": "many miles", "item": "most expensive white sports car in the \"Cars + trucks\" category from Ohio" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "36K |OR| 36,000 |OR| 36000" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=48555", "intent_template_id": 9 }, { "sites": [ "classifieds" ], "task_id": 110, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and tell me {{detail}}.", "intent": "Search for \"mario kart\" and tell me how many games are in the image of the most recently listed item that costs at least $300.", "image": null, "instantiation_dict": { "phrase": "mario kart", "detail": "how many games are in the image of the most recently listed item that costs at least $300" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "0 |OR| zero" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=34406", "intent_template_id": 16 }, { "sites": [ "classifieds" ], "task_id": 111, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and tell me {{detail}}.", "intent": "Search for \"hockey\" and tell me the team name of the jersey on the most recently listed item.", "image": null, "instantiation_dict": { "phrase": "hockey", "detail": "the team name of the jersey on the most recently listed item" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Kings" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=79079", "intent_template_id": 16 }, { "sites": [ "classifieds" ], "task_id": 112, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}} that {{condition}} in its image.", "intent": "Search for \"basketball\" and navigate to the cheapest item that has a man in a suit in its image.", "image": null, "instantiation_dict": { "phrase": "basketball", "product": "cheapest item", "condition": "has a man in a suit" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=68564", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 41 }, { "sites": [ "classifieds" ], "task_id": 113, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}}.", "intent": "Search for \"football\" and navigate to the most expensive item whose image has a player wearing number 16.", "image": null, "instantiation_dict": { "phrase": "football", "product": "most expensive item whose image has a player wearing number 16" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=30630", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 42 }, { "sites": [ "classifieds" ], "task_id": 114, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{item}} from {{location}} {{condition}}.", "intent": "Navigate to the oldest listing from Virginia which contains multiple people in its image.", "image": null, "instantiation_dict": { "item": "oldest listing", "location": "Virginia", "condition": "which contains multiple people in its image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=29164", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 43 }, { "sites": [ "classifieds" ], "task_id": 115, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}} that matches the body part in this image.", "intent": "Search for \"brace\" and navigate to the most recently listed item that matches the body part in this image.", "image": "https://images.pexels.com/photos/356175/pexels-photo-356175.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "phrase": "brace", "product": "most recently listed item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=68133", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 44 }, { "sites": [ "classifieds" ], "task_id": 116, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}} that matches the body part in this image.", "intent": "Search for \"brace\" and navigate to the cheapest item that matches the body part in this image.", "image": "https://images.pexels.com/photos/2077959/pexels-photo-2077959.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "phrase": "brace", "product": "cheapest item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=42654", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 44 }, { "sites": [ "classifieds" ], "task_id": 117, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest bike which is of a similar color as the person's outfit.", "image": "https://images.pexels.com/photos/14656911/pexels-photo-14656911.jpeg?auto=compress&cs=tinysrgb&w=800", "instantiation_dict": { "item": "bike", "condition": "which is of a similar color as the person's outfit" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=18534", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 118, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=11&sOrder=dt_pub_date&iOrderType=desc&sShowAs=gallery", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is {{attribute}} on the {{product}}?", "intent": "What is the time on the phone with the visible screen in the third row on this page?", "image": null, "instantiation_dict": { "attribute": "the time", "product": "phone with the visible screen in the third row on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "3:03 |OR| 303" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=69495", "intent_template_id": 38 }, { "sites": [ "classifieds" ], "task_id": 119, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=11&sOrder=i_price&iOrderType=desc&sShowAs=gallery", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is the {{attribute}} of {{item}}?", "intent": "What is the denomination of the dollar bill shown in that image on this page?", "image": null, "instantiation_dict": { "attribute": "denomination", "item": "the dollar bill shown in that image on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "50 |OR| fifty" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "_CLASSIFIEDS__/index.php?page=item&id=75385", "intent_template_id": 5 }, { "sites": [ "classifieds" ], "task_id": 120, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=7&sShowAs=gallery&sOrder=i_price&iOrderType=desc", "geolocation": null, "viewport_size": { "width": 1280 }, "intent_template": "What is {{attribute}} on the {{product}}?", "intent": "What is written on the down tube of the bike with the purple frame on this page?", "image": null, "instantiation_dict": { "attribute": "written", "product": "down tube of the bike with the purple frame on this page" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "canyon" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=38276", "intent_template_id": 38 }, { "sites": [ "classifieds" ], "task_id": 121, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=11&sOrder=i_price&iOrderType=asc&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page {{condition}}.", "intent": "Navigate to the item on this page whose image consists solely of text.", "image": null, "instantiation_dict": { "item": "item", "condition": "whose image consists solely of text" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=15619", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 45 }, { "sites": [ "classifieds" ], "task_id": 122, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}}, and tell me how much it costs in dollars.", "intent": "Find the oldest listed red book in the \"Books\" category, and tell me how much it costs in dollars.", "image": null, "instantiation_dict": { "product": "oldest listed red book in the \"Books\" category" }, "require_reset": false, "eval": { "eval_types": [ "url_match", "string_match" ], "reference_answers": { "must_include": [ "3" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=69456", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 33 }, { "sites": [ "classifieds" ], "task_id": 123, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sPattern=clothes&sOrder=i_price&iOrderType=desc&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page {{condition}}.", "intent": "Navigate to the item on this page which matches the color of the object being used by the person in the given image.", "image": "https://images.pexels.com/photos/13991925/pexels-photo-13991925.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "item", "condition": "which matches the color of the object being used by the person in the given image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=48074", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 45 }, { "sites": [ "classifieds" ], "task_id": 124, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=17&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image is set on grass.", "image": null, "instantiation_dict": { "condition": "is set on grass" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=10702", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 125, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=17&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image is a desktop screenshot.", "image": null, "instantiation_dict": { "condition": "is a desktop screenshot" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=32204", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 126, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me {{product}} that {{condition}} in the image.", "intent": "Find me the most recently listed item in the \"Books\" category that references the person in the image.", "image": "https://images.pexels.com/photos/3592123/pexels-photo-3592123.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "the most recently listed item in the \"Books\" category", "condition": "references the person" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=64659", "program_html": [], "url_note": "EXACT" }, "intent_template_id": 30, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "" }, { "sites": [ "classifieds" ], "task_id": 127, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}}.", "intent": "Search for \"MCAT\" and navigate to the prep book that has 2020-2021 on the cover.", "image": null, "instantiation_dict": { "phrase": "MCAT", "product": "prep book that has 2020-2021 on the cover" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=5802", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 42 }, { "sites": [ "classifieds" ], "task_id": 128, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=74603", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the jersey numbers (in digits) in the image of this listing.", "image": null, "instantiation_dict": { "product": "jersey numbers (in digits) in the image of this listing" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "99", "13", "80", "92" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 129, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sOrder=dt_pub_date&iOrderType=desc&sPattern=soccer+cleats&sShowAs=gallery", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the last name of the seller whose listing image on the page shows the price.", "image": null, "instantiation_dict": { "product": "last name of the seller whose listing image on the page shows the price" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Chatterjee" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 130, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sOrder=dt_pub_date&iOrderType=desc&sPattern=banana+boat&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image is taken during a sunset.", "image": null, "instantiation_dict": { "condition": "is taken during a sunset" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=19604", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 131, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and navigate to the {{product}}.", "intent": "Search for \"dogs\" and navigate to the listing whose image consists of puppies in a basket.", "image": null, "instantiation_dict": { "phrase": "dogs", "product": "listing whose image consists of puppies in a basket" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=50134", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 42 }, { "sites": [ "classifieds" ], "task_id": 132, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sOrder=dt_pub_date&iOrderType=desc&sPattern=dogs&sShowAs=gallery&iPage=3", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image has 3 black animals.", "image": null, "instantiation_dict": { "condition": "has 3 black animals" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=69201", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 133, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} with {{attribute}}. Add a {{rating}} rating with title \"{{title}}\" and text \"{{text}}\".", "intent": "Find the oldest listing of Xbox One games in the \"Video gaming\" category with more than 9 games in the listing image. Add a 4 star rating with title \"Nice collection\" and text \"Any other games?\".", "image": null, "instantiation_dict": { "item": "oldest listing of Xbox One games in the \"Video gaming\" category", "attribute": "more than 9 games in the listing image", "rating": "4 star", "title": "Nice collection", "text": "Any other games?" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=67441", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Nice collection by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=67441", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Any other games?", "4 of 5" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 35 }, { "sites": [ "classifieds" ], "task_id": 134, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sOrder=dt_pub_date&iOrderType=desc&sPattern=+Xbox+One+games&sCategory=24&iPage=5&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} which {{condition}} in this image.", "intent": "Navigate to the listing on this page whose image includes a jersey which is the same color as the item in this image.", "image": "https://images.pexels.com/photos/36753/flower-purple-lical-blosso.jpg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "listing on this page whose image includes a jersey", "condition": "is the same color as the item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=42088", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 47 }, { "sites": [ "classifieds" ], "task_id": 135, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sOrder=dt_pub_date&iOrderType=desc&sPattern=hood&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} on this page {{condition}}.", "intent": "Navigate to the listing on this page with an item of the exact same color as the one in this image.", "image": "https://images.pexels.com/photos/18756727/pexels-photo-18756727/free-photo-of-mistry-of-the-hot-pink-mclaren.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "listing", "condition": "with an item of the exact same color as the one in this image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no pink items on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 48 }, { "sites": [ "classifieds" ], "task_id": 136, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, tell me the email of the lister of the item. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, tell me the email of the lister of the item. I recall that it was pretty cheap.", "image": "__CLASSIFIEDS__/oc-content/uploads/37999/37999.png", "instantiation_dict": { "hint": "that it was pretty cheap" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "noah_popov126@example.com" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=37999", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 49 }, { "sites": [ "classifieds" ], "task_id": 137, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall that it cost $20.", "image": "__CLASSIFIEDS__/oc-content/uploads/49213/49213.png", "instantiation_dict": { "hint": "that it cost $20" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "mia_robinson403@example.com" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=19390", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 50 }, { "sites": [ "classifieds" ], "task_id": 138, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall it was in the Art + crafts section and between $600-$1000.", "image": "__CLASSIFIEDS__/oc-content/uploads/70697/70697.png", "instantiation_dict": { "hint": "it was in the Art + crafts section and between $600-$1000" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "sarah.sidorov92@example.com" ] }, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=70697", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 50 }, { "sites": [ "classifieds" ], "task_id": 139, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall it was in the Antiques section.", "image": "__CLASSIFIEDS__/oc-content/uploads/47595/47595.png", "instantiation_dict": { "hint": "it was in the Antiques section" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=47595", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 19 }, { "sites": [ "classifieds" ], "task_id": 140, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, tell me the email of the most recent lister of the item. I recall it was a Nascar jacket in the Collectibles section.", "image": "__CLASSIFIEDS__/oc-content/uploads/3984/3984.png", "instantiation_dict": { "hint": "it was a Nascar jacket in the Collectibles section" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "juan_ortiz@example.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=3984", "intent_template_id": 50 }, { "sites": [ "classifieds" ], "task_id": 141, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall {{hint}}.", "intent": "I recall seeing this exact item on the site, help me find the most recent post of it. I recall it was in the Farm + garden section.", "image": "__CLASSIFIEDS__/oc-content/uploads/41568/41568.png", "instantiation_dict": { "hint": "it was in the Farm + garden section" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=41568", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 19 }, { "sites": [ "classifieds" ], "task_id": 142, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact item for under $20 on the site, tell me the state where this listing is located.", "intent": "I recall seeing this exact item for under $20 on the site, tell me the state where this listing is located.", "image": "__CLASSIFIEDS__/oc-content/uploads/22310/22310.png", "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Pennsylvania" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=22310", "intent_template_id": 51 }, { "sites": [ "classifieds" ], "task_id": 143, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact {{item}} on the site, add a comment on its listing with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "I recall seeing this exact item for under $15 in the Arts+ crafts section on the site, add a comment on its listing with the title \"Inquiry\" and text \"I need this!\".", "image": "__CLASSIFIEDS__/oc-content/uploads/25810/25810.png", "instantiation_dict": { "item": "item for under $15 in the Arts+ crafts section", "title": "Inquiry", "comment": "I need this!" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=25810", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Inquiry by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=25810", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "I need this!" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=25810", "intent_template_id": 52 }, { "sites": [ "classifieds" ], "task_id": 144, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact {{item}} on the site, add a comment on its listing with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "I recall seeing this exact item of pillows in the Household section on the site, add a comment on its listing with the title \"Commentary\" and text \"How funky...\".", "image": "__CLASSIFIEDS__/oc-content/uploads/41127/41127.png", "instantiation_dict": { "item": "item of pillows in the Household section", "title": "Commentary", "comment": "How funky..." }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=41127", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Commentary by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=41127", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "How funky..." ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=41127", "intent_template_id": 52 }, { "sites": [ "classifieds" ], "task_id": 145, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "I recall seeing this exact {{item}} on the site, add a comment on its listing with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "I recall seeing this exact expensive item in the Video Gaming category on the site, add a comment on its listing with the title \"Urgent\" and text \"Need this for my bowling alley.\".", "image": "__CLASSIFIEDS__/oc-content/uploads/43992/43992.png", "instantiation_dict": { "item": "expensive item in the Video Gaming category", "title": "Urgent", "comment": "Need this for my bowling alley." }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=43992", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Urgent by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=43992", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Need this for my bowling alley." ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=43992", "intent_template_id": 52 }, { "sites": [ "classifieds" ], "task_id": 146, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sOrder=i_price&iOrderType=asc&iPage=124&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} on this page {{condition}}.", "intent": "Navigate to the item on this page whose image has exactly 4 books.", "image": null, "instantiation_dict": { "product": "item", "condition": "whose image has exactly 4 books" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66304", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 48 }, { "sites": [ "classifieds" ], "task_id": 147, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=24&sOrder=i_price&iOrderType=desc&iPage=14&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} on this page {{condition}}.", "intent": "Navigate to the item on this page whose image has multiple game consoles from different brands.", "image": null, "instantiation_dict": { "product": "item", "condition": "whose image has multiple game consoles from different brands" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=35394", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 48 }, { "sites": [ "classifieds" ], "task_id": 148, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=24&sOrder=i_price&iOrderType=desc&iPage=17&sShowAs=gallery", "geolocation": null, "intent_template": "For the {{item}}, tell me the {{detail}}.", "intent": "For the item with the Amazon webpage in its image, tell me the number of ratings the item on the page has.", "image": null, "instantiation_dict": { "item": "item with the Amazon webpage in its image", "detail": "number of ratings the item on the page has" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "128 |OR| hundred twenty eight |OR| one hundred twenty eight" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=33746", "intent_template_id": 53 }, { "sites": [ "classifieds" ], "task_id": 149, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=24&sOrder=i_price&iOrderType=desc&iPage=22&sShowAs=gallery", "geolocation": null, "intent_template": "For the {{item}}, tell me the {{detail}}.", "intent": "For the item with a gold colored controller, tell me the number of games displayed in the image.", "image": null, "instantiation_dict": { "item": "item with a gold colored controller", "detail": "number of games displayed in the image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "12 |OR| twelve" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=53116", "intent_template_id": 53 }, { "sites": [ "classifieds" ], "task_id": 150, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=16&sOrder=i_price&iOrderType=asc&iPage=331&sShowAs=gallery", "geolocation": null, "intent_template": "For the {{item}}, tell me the {{detail}}.", "intent": "For the item with the Amazon webpage in its image, tell me the price in dollars displayed on the Amazon site.", "image": null, "instantiation_dict": { "item": "item with the Amazon webpage in its image", "detail": "price in dollars displayed on the Amazon site" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "249 |OR| two hundred twenty nine" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=9246", "intent_template_id": 53 }, { "sites": [ "classifieds" ], "task_id": 151, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&iPage=2&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image has pink sticky notes.", "image": null, "instantiation_dict": { "condition": "has pink sticky notes" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=22560", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 152, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=22&sShowAs=gallery&iPage=11", "geolocation": null, "intent_template": "Navigate to the item on this page whose image {{condition}}.", "intent": "Navigate to the item on this page whose image includes a human hand.", "image": null, "instantiation_dict": { "condition": "includes a human hand" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=81346", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 46 }, { "sites": [ "classifieds" ], "task_id": 153, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/php?page=search&sCategory=4&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page {{condition}}.", "intent": "Navigate to the item on this page that most similarly depicts the setting in the image.", "image": "https://images.pexels.com/photos/9344514/pexels-photo-9344514.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "item", "condition": "that most similarly depicts the setting in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=14761", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 45 }, { "sites": [ "classifieds" ], "task_id": 154, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Furniture\" category, and find the oldest listed item whose image is taken outside.", "image": null, "instantiation_dict": { "category": "Furniture", "item": "oldest listed item whose image is taken outside" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=67923", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 155, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Computers\" category, and find the oldest listed item whose image shows a Microsoft Windows background.", "image": null, "instantiation_dict": { "category": "Computers", "item": "oldest listed item whose image shows a Microsoft Windows background" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=14744", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 156, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Bikes\" category, and find the oldest listed item whose image is taken on carpet.", "image": null, "instantiation_dict": { "category": "Bikes", "item": "oldest listed item whose image is taken on carpet" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6429", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 157, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Music instruments\" category, and find the oldest listed item whose image shows an instruction manual.", "image": null, "instantiation_dict": { "category": "Music instruments", "item": "oldest listed item whose image shows an instruction manual" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=33917", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 158, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Jewelry\" category, and find the oldest listed necklace whose image shows the necklace being worn by a human.", "image": null, "instantiation_dict": { "category": "Jewelry", "item": "oldest listed necklace whose image shows the necklace being worn by a human" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=69302", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 159, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to my listing {{item}} and {{action}}.", "intent": "Navigate to my listing with a rug in the image and add to the description that there are no refunds.", "image": null, "instantiation_dict": { "item": "with a rug in the image", "action": "add to the description that there are no refunds" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=84154", "locator": "func:get_query_text(__page__, '.desc')", "required_contents": { "must_include": [ "no refund |OR| no refunds |OR| 0 refund |OR| zero refunds" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 2 }, { "sites": [ "classifieds" ], "task_id": 160, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=5&sShowAs=gallery", "geolocation": null, "intent_template": "Add a comment on the {{item}} with the title \"{{title}}\" and text \"{{comment}}\".", "intent": "Add a comment on the item on this page whose image is taken inside a garage with the title \"More Pictures\" and text \"Could you post different angles?\".", "image": null, "instantiation_dict": { "item": "item on this page whose image is taken inside a garage", "title": "More Pictures", "comment": "Could you post different angles?" }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=13412", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "More Pictures by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=13412", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Could you post different angles?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=13412", "intent_template_id": 11 }, { "sites": [ "classifieds" ], "task_id": 161, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/", "geolocation": null, "intent_template": "Search for \"{{phrase}}\" and tell me {{detail}}.", "intent": "Search for \"mountain bike\" and tell me the predominant color of the frame of the cheapest bike, ignoring $1 listings.", "image": null, "instantiation_dict": { "phrase": "mountain bike", "detail": "the predominant color of the frame of the cheapest bike, ignoring $1 listings" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "purple" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "mediun", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=22441", "intent_template_id": 16 }, { "sites": [ "classifieds" ], "task_id": 162, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/", "geolocation": null, "intent_template": "Explore the \"{{category}}\" category, and find the {{item}}.", "intent": "Explore the \"Beauty + Health\" category, and find the most recent item that shows a tennis ball.", "image": null, "instantiation_dict": { "category": "Beauty + Health", "item": "most recent item that shows a tennis ball" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=61809", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 36 }, { "sites": [ "classifieds" ], "task_id": 163, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=18", "geolocation": null, "intent_template": "For the {{item}} on this page {{attribute}}, tell me the {{detail}}.", "intent": "For the item on this page with the Amazon webpage in its image, tell me the price in dollars displayed on the Amazon site.", "image": null, "instantiation_dict": { "item": "item", "attribute": "with the Amazon webpage in its image", "detail": "price in dollars displayed on the Amazon site" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "40.92" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=52143", "intent_template_id": 54 }, { "sites": [ "classifieds" ], "task_id": 164, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&iPage=4y", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page whose image {{condition}} as the item in the image provided.", "intent": "Navigate to the listing on this page whose image includes a car the same color as the item in the image provided.", "image": "https://images.pexels.com/photos/2872755/pexels-photo-2872755.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "listing", "condition": "includes a car the same color" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no yellow items on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 55 }, { "sites": [ "classifieds" ], "task_id": 165, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=10&iPage=8", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page whose image {{condition}} as the item in the image provided.", "intent": "Navigate to the listing on this page whose image includes a car the same color as the item in the image provided.", "image": "https://images.pexels.com/photos/6944172/pexels-photo-6944172.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "listing", "condition": "includes a car the same color" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=34939", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 55 }, { "sites": [ "classifieds" ], "task_id": 166, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=21&iPage=11", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page whose image {{condition}} as the item in the image provided.", "intent": "Navigate to the listing on this page whose image includes an instrument the same color as the item in the image provided.", "image": "https://images.pexels.com/photos/1153655/pexels-photo-1153655.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "listing", "condition": "includes an instrument the same color" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=40109", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 55 }, { "sites": [ "classifieds" ], "task_id": 167, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=21&iPage=8", "geolocation": null, "intent_template": "Navigate to the {{item}} on this page whose image {{condition}} as the item in the image provided.", "intent": "Navigate to the listing on this page whose image includes an instrument the same color as the item in the image provided.", "image": "https://images.pexels.com/photos/693794/pexels-photo-693794.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "listing", "condition": "includes an instrument the same color" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no green instruments on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 55 }, { "sites": [ "classifieds" ], "task_id": 168, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the cheapest {{item}} {{condition}}.", "intent": "Find the cheapest motorcycle which is of the same color as the item in the image provided.", "image": "https://images.pexels.com/photos/2737656/pexels-photo-2737656.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "item": "motorcycle", "condition": "which is of the same color as the item in the image provided" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=45405", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 32 }, { "sites": [ "classifieds" ], "task_id": 169, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} that costs {{range}}.", "intent": "Navigate to the cheapest listing in the \"Cell phones\" category with a black phone that costs more than $1.", "image": null, "instantiation_dict": { "product": "cheapest listing in the \"Cell phones\" category with a black phone", "range": "more than $1" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=50277", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "medium", "comments": "", "intent_template_id": 56 }, { "sites": [ "classifieds" ], "task_id": 170, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} which {{condition}} in this image.", "intent": "Navigate to the most expensive item in the \"Cell phones\" category which includes in its design the character in this image.", "image": "https://images.pexels.com/photos/9482194/pexels-photo-9482194.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most expensive item in the \"Cell phones\" category", "condition": "includes in its design the character" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=36313", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 47 }, { "sites": [ "classifieds" ], "task_id": 171, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}}.", "intent": "Navigate to the oldest listing in the \"Cell phones\" category which has a hot pink screen in the image.", "image": null, "instantiation_dict": { "product": "oldest listing in the \"Cell phones\" category which has a hot pink screen in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=32832", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 57 }, { "sites": [ "classifieds" ], "task_id": 172, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}}.", "intent": "Navigate to the cheapest listing in the \"Furniture category\" with three blue chairs suitable for a workplace in the image.", "image": null, "instantiation_dict": { "product": "cheapest listing in the \"Furniture category\" with three blue chairs suitable for a workplace in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=23701", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 57 }, { "sites": [ "classifieds" ], "task_id": 173, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=17&sOrder=i_price&iOrderType=desc&sShowAs=gallery", "geolocation": null, "intent_template": "For the {{item}} on this page {{attribute}}, tell me the {{detail}}.", "intent": "For the item on this page with a website address in the image, tell me the address.", "image": null, "instantiation_dict": { "item": "item", "attribute": "with a website address in the image", "detail": "address" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "kaiyo.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 54 }, { "sites": [ "classifieds" ], "task_id": 174, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=17&sOrder=i_price&iOrderType=desc&sShowAs=gallery", "geolocation": null, "intent_template": "For the {{item}} on this page {{attribute}}, tell me the {{detail}}.", "intent": "For the item on this page which includes a Black Friday logo in the image, tell me the most specific location given of the posting.", "image": null, "instantiation_dict": { "item": "item", "attribute": "which includes a Black Friday logo in the image", "detail": "most specific location given of the posting" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "Shaw", "Washington, D.C." ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 54 }, { "sites": [ "classifieds" ], "task_id": 175, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} from {{location}} that {{condition}}.", "intent": "Navigate to the cheapest controller in the \"Video gaming\" category from Ohio or Pennsylvania that is predominantly white.", "image": null, "instantiation_dict": { "product": "cheapest controller in the \"Video gaming\" category", "location": "Ohio or Pennsylvania", "condition": "is predominantly white" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=80517", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 58 }, { "sites": [ "classifieds" ], "task_id": 176, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} from {{location}} that {{condition}}.", "intent": "Navigate to the most expensive motorcycle from Virginia or Maryland that has both red and blue colors.", "image": null, "instantiation_dict": { "product": "most expensive motorcycle", "location": "Virginia or Maryland", "condition": "has both red and blue colors" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=78122", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 58 }, { "sites": [ "classifieds" ], "task_id": 177, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} from {{location}} that {{condition}}.", "intent": "Navigate to the cheapest listing in the \"Cars + trucks\" category from Ohio or Maryland that is the same color as the objects in this image.", "image": "https://images.pexels.com/photos/1179229/pexels-photo-1179229.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "cheapest listing in the \"Cars + trucks\" category", "location": "Ohio or Maryland", "condition": "is the same color as the objects in this image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=15340", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "easy", "overall_difficulty": "hard", "comments": "", "intent_template_id": 58 }, { "sites": [ "classifieds" ], "task_id": 178, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "For the {{item}}, tell me the {{detail}}.", "intent": "For the cheapest listing in the \"Music instruments\" category with a white guitar from West Virginia or Delaware, tell me the price in dollars.", "image": null, "instantiation_dict": { "item": "cheapest listing in the \"Music instruments\" category with a white guitar from West Virginia or Delaware", "detail": "price in dollars" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "41 |OR| fourty one" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 53 }, { "sites": [ "classifieds" ], "task_id": 179, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{product}} with {{detail}} from the image {{location}}.", "intent": "Find the most expensive item in the \"Video gaming\" category with the character on the shirt from the image on its decal.", "image": "https://images.pexels.com/photos/2560894/pexels-photo-2560894.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most expensive item in the \"Video gaming\" category", "detail": "the character on the shirt", "location": "on its decal" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6909", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 25 }, { "sites": [ "classifieds" ], "task_id": 180, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} with {{attribute}}. Add a {{rating}} rating with title \"{{title}}\" and text \"{{text}}\".", "intent": "Find the most recently listed coffee maker with a touch screen. Add a 5 star rating with title \"Great item\" and text \"Would recommend!\".", "image": null, "instantiation_dict": { "item": "most recently listed coffee maker", "attribute": "a touch screen", "rating": "5 star", "title": "Great item", "text": "Would recommend!" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=15810", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Great item by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=15810", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Would recommend!", "5 of 5" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 35 }, { "sites": [ "classifieds" ], "task_id": 181, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} with {{detail}} from the image.", "intent": "Navigate to the most expensive item in the \"Video gaming\" category with the same item from the image.", "image": "https://images.pexels.com/photos/33571/tutankhamun-death-mask-pharaonic-egypt.jpg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most expensive item in the \"Video gaming\" category", "detail": "the same item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=7915", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 59 }, { "sites": [ "classifieds" ], "task_id": 182, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}}.", "intent": "Navigate to the most recent video game listing with the the Playstation home screen in the image.", "image": null, "instantiation_dict": { "product": "most recent video game listing with the the Playstation home screen in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=41845", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 57 }, { "sites": [ "classifieds" ], "task_id": 183, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} with {{detail}} from the image.", "intent": "Navigate to the most recently listed item in the \"Collectibles\" category with the same character from the image.", "image": "https://images.pexels.com/photos/12695401/pexels-photo-12695401.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most recently listed item in the \"Collectibles\" category", "detail": "the same character" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=40185", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "medium", "overall_difficulty": "hard", "comments": "", "intent_template_id": 59 }, { "sites": [ "classifieds" ], "task_id": 184, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} with {{detail}} from the image.", "intent": "Navigate to the most expensive item in the \"Collectibles\" category with the same exact item from the image.", "image": "https://images.pexels.com/photos/9661257/pexels-photo-9661257.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "most expensive item in the \"Collectibles\" category", "detail": "the same exact item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=18837", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 59 }, { "sites": [ "classifieds" ], "task_id": 185, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me {{product}} that {{condition}} in the image.", "intent": "Find me the most expensive Collectibles listing that has a listing image that references a sports team for the city in the image.", "image": "https://images.pexels.com/photos/1750754/pexels-photo-1750754.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "the most expensive Collectibles listing", "condition": "has a listing image that references a sports team for the city" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=33317", "program_html": [], "url_note": "EXACT" }, "intent_template_id": 30, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "" }, { "sites": [ "classifieds" ], "task_id": 186, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "For the {{item}}, tell me the {{detail}}.", "intent": "For the most recently listed item in the \"Books\" category of a single book with a car on the cover, tell me the color of the car.", "image": null, "instantiation_dict": { "item": "most recently listed item in the \"Books\" category of a single book with a car on the cover", "detail": "color of the car" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "yellow" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "__CLASSIFIEDS__/index.php?page=item&id=9146", "intent_template_id": 53 }, { "sites": [ "classifieds" ], "task_id": 187, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&iPage=6&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} where {{detail}}.", "intent": "Navigate to the item on the page where Lightning McQueen is in the image.", "image": null, "instantiation_dict": { "product": "item on the page", "detail": "Lightning McQueen is in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=79133", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 60 }, { "sites": [ "classifieds" ], "task_id": 188, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&iPage=14&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} where {{detail}}.", "intent": "Navigate to the book listing on the page where the cover includes a baby.", "image": null, "instantiation_dict": { "product": "book listing on the page", "detail": "the cover includes a baby" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=55076", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 60 }, { "sites": [ "classifieds" ], "task_id": 189, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=24&sShowAs=gallery&iPage=5", "geolocation": null, "intent_template": "Navigate to the {{product}} where {{detail}}.", "intent": "Navigate to the item on the page where Mario is in the image.", "image": null, "instantiation_dict": { "product": "item on the page", "detail": "Mario is in the image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no items with Mario on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 60 }, { "sites": [ "classifieds" ], "task_id": 190, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=24&sShowAs=gallery&iPage=5", "geolocation": null, "intent_template": "Navigate to the {{product}} with {{detail}} from the image {{location}}.", "intent": "Navigate to the cheapest video game item on this page with the character on the shirt from the image in its listing image.", "image": "https://images.pexels.com/photos/2560894/pexels-photo-2560894.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "product": "cheapest video game item on this page", "detail": "the character on the shirt", "location": "in its listing image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=29518", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 61 }, { "sites": [ "classifieds" ], "task_id": 191, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=16&sOrder=dt_pub_date&iOrderType=desc&iPage=3&sShowAs=gallery", "geolocation": null, "intent_template": "Navigate to the {{product}} on this page {{condition}}.", "intent": "Navigate to the item on this page with a blonde horse.", "image": null, "instantiation_dict": { "product": "item", "condition": "with a blonde horse" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no items with a blonde horse on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 48 }, { "sites": [ "classifieds" ], "task_id": 192, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sShowAs=gallery&sOrder=i_price&iOrderType=asc&iPage=119", "geolocation": null, "intent_template": "For the {{item}} on this page {{attribute}}, tell me the {{detail}}.", "intent": "For the listing on this page with books of cars, tell me the primary color of the cars.", "image": null, "instantiation_dict": { "item": "listing", "attribute": "with books of cars", "detail": "primary color of the cars" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "red", "white" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "__CLASSIFIEDS__/index.php?page=item&id=57831", "intent_template_id": 54 }, { "sites": [ "classifieds" ], "task_id": 193, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sShowAs=gallery&sOrder=dt_pub_date&iOrderType=desc&iPage=106", "geolocation": null, "intent_template": "Navigate to the {{product}} on this page {{condition}}.", "intent": "Navigate to the item on this page with CDs in the image.", "image": null, "instantiation_dict": { "product": "item", "condition": "with CDs in the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=2753", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "easy", "comments": "", "intent_template_id": 48 }, { "sites": [ "classifieds" ], "task_id": 194, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=search&sCategory=9&sShowAs=gallery&sOrder=dt_pub_date&iOrderType=desc&iPage=90", "geolocation": null, "intent_template": "Navigate to the {{product}} where {{detail}}.", "intent": "Navigate to the item on the page where Mickey Mouse is in the image.", "image": null, "instantiation_dict": { "product": "item on the page", "detail": "Mickey Mouse is in the image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no items with Mickey Mouse on the page.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "", "intent_template_id": 60 }, { "sites": [ "classifieds" ], "task_id": 195, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} that costs {{range}}.", "intent": "Navigate to the most expensive yellow motorcycle in the \"Motorcycles\" category that costs at least 25,000 dollars.", "image": null, "instantiation_dict": { "product": "most expensive yellow motorcycle in the \"Motorcycles\" category", "range": "at least 25,000 dollars" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no yellow motorcycles worth at least 25000 dollars.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 56 }, { "sites": [ "classifieds" ], "task_id": 196, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} from {{location}} that {{condition}}.", "intent": "Navigate to the most expensive red truck in the \"Cars + trucks\" category from Maryland that is at least 50,000 dollars.", "image": null, "instantiation_dict": { "product": "most expensive red truck in the \"Cars + trucks\" category", "location": "Maryland", "condition": "is at least 50,000 dollars" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "There are no red trucks from Maryland worth at least 50000 dollars.", "reference_answer_raw_annotation": "N/A" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 58 }, { "sites": [ "classifieds" ], "task_id": 197, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} from {{location}} that {{condition}}.", "intent": "Navigate to the cheapest boat in the \"Boats\" category from Pennsylvania that has red highlights and is at least 30,000 dollars.", "image": null, "instantiation_dict": { "product": "cheapest boat in the \"Boats\" category", "location": "Pennsylvania", "condition": "has red highlights and is at least 30,000 dollars" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=3256", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 58 }, { "sites": [ "classifieds" ], "task_id": 198, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Navigate to the {{product}} that costs {{range}}.", "intent": "Navigate to the cheapest watch with hints of faded red and green that costs at least 13,000 dollars.", "image": null, "instantiation_dict": { "product": "cheapest watch with hints of faded red and green", "range": "at least 13,000 dollars" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=37012", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 56 }, { "sites": [ "classifieds" ], "task_id": 199, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=43887", "geolocation": null, "intent_template": "Tell me the {{product}}.", "intent": "Tell me the website mentioned in the image.", "image": null, "instantiation_dict": { "product": "website mentioned in the image" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "kaiyo.com" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 40 }, { "sites": [ "classifieds" ], "task_id": 200, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__SHOPPING__/24-pcs-lot-wild-animal-cupcake-wrappers-birthday-cake-party-decoration-supplies-lion-monkey-zebra-elephant.html |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the latest listing of a pillow from the classifieds site with a pattern matching the top left animal in the first image of the listing on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "latest", "item": "listing of a pillow", "description": "with a pattern matching the top left animal in the first image of the listing" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=65205", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 201, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the latest listing of a snare drum with a black and red color scheme.", "image": null, "instantiation_dict": { "attribute": "latest", "item": "listing of a snare drum", "condition": "with a black and red color scheme" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=50230", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 202, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the cheapest penguin-themed item in West Virginia.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "penguin-themed item", "condition": "in West Virginia" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=6247", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 203, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=41938", "geolocation": null, "intent_template": "Help me leave a comment with the title \"Interested\" with the text \"I want to buy this item\", if the item comes with a cable that is able to connect to my USB-C ports, else leave a comment with the same title but with the text \"Do you have a USB-C cable?\".", "intent": "Help me leave a comment with the title \"Interested\" with the text \"I want to buy this item\", if the item comes with a cable that is able to connect to my USB-C ports, else leave a comment with the same title but with the text \"Do you have a USB-C cable?\".", "image": null, "instantiation_dict": {}, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=41938", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interested by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=41938", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Do you have a USB-C cable?" ] } } ] }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 64 }, { "sites": [ "classifieds" ], "task_id": 204, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} that {{condition}}.", "intent": "Find the cheapest guitar between $1200 and $1500 that is made of lightly-colored wood and comes with a soft guitar case that has blue-colored insides.", "image": null, "instantiation_dict": { "item": "cheapest guitar between $1200 and $1500", "condition": "is made of lightly-colored wood and comes with a soft guitar case that has blue-colored insides" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=76469", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 23 }, { "sites": [ "classifieds" ], "task_id": 205, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=65673 |AND| __CLASSIFIEDS__/index.php?page=item&id=43183", "geolocation": null, "intent_template": "Which palette is more value-for-money in terms of price per distinct color? Help me make a comment on that one with the title \"Would like to buy\" and text \"Do you accept PayPal?\".", "intent": "Which palette is more value-for-money in terms of price per distinct color? Help me make a comment on that one with the title \"Would like to buy\" and text \"Do you accept PayPal?\".", "image": null, "instantiation_dict": {}, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=43183", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Would like to buy by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=43183", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Do you accept PayPal?" ] } } ] }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 65 }, { "sites": [ "classifieds" ], "task_id": 206, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} that {{condition}}.", "intent": "Find the cheapest freezer with a warranty between $250 and $280 that does not have a fridge.", "image": null, "instantiation_dict": { "item": "cheapest freezer with a warranty between $250 and $280", "condition": "does not have a fridge" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=32949", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 23 }, { "sites": [ "classifieds" ], "task_id": 207, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__SHOPPING__/nintendo-switch-oled-model-w-neon-red-neon-blue-joy-con.html |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the cheapest headphones from the classifieds site that matches the color scheme of the item on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "headphones", "description": "that matches the color scheme of the item" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=44307", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 208, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=16826", "geolocation": null, "intent_template": "Identify the insect in the picture. Leave a comment with the title \"Questions\" and text containing the insect's identity, with the purpose of confirming with the seller.", "intent": "Identify the insect in the picture. Leave a comment with the title \"Questions\" and text containing the insect's identity, with the purpose of confirming with the seller.", "image": null, "instantiation_dict": {}, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=16826", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Questions by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=16826", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "moth |OR| butterfly" ] } } ] }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 66 }, { "sites": [ "classifieds" ], "task_id": 209, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=27156", "geolocation": null, "intent_template": "What is the {{detail}} that {{product}}?", "intent": "What is the cheapest price that I can pay for 31 of these wooden pallets?", "image": null, "instantiation_dict": { "detail": "cheapest price", "product": "I can pay for 31 of these wooden pallets" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "$208" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "medium", "comments": "Buying in batches 15 is cheaper, a fact which is only spelled out in the image and not the description", "intent_template_id": 67 }, { "sites": [ "classifieds" ], "task_id": 210, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} on this site.", "intent": "Find me the cheapest lamb in the \"Farm + garden\" category on this site.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "lamb in the \"Farm + garden\" category" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=81060", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "medium", "visual_difficulty": "medium", "overall_difficulty": "medium", "comments": "", "intent_template_id": 0 }, { "sites": [ "classifieds" ], "task_id": 211, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} on this site.", "intent": "Find me the cheapest farm pig in the \"Farm + garden\" category on this site.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "farm pig in the \"Farm + garden\" category" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=72643", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 0 }, { "sites": [ "classifieds" ], "task_id": 212, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} from the classifieds site that {{condition}}.{{action}}", "intent": "Find me the cheapest watch or necklace from the classifieds site that is from the location \"Pennsylvania\". Do not take the listing price at face value and use the actual item price for comparison, which can be verified in the listing description.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "watch or necklace", "condition": "is from the location \"Pennsylvania\"", "action": " Do not take the listing price at face value and use the actual item price for comparison, which can be verified in the listing description." }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=47839", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "The other watch/ring listings on the website are fake listings that state their actual price in the description. Super hard task.", "intent_template_id": 68 }, { "sites": [ "classifieds" ], "task_id": 213, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=79640", "geolocation": null, "intent_template": "Find me the seller's email if the item depicts Elvis Presley's performance without the audience shown, else leave a comment with the title \"Question\" and text \"Do you have one without the audience?\". Also, repeat the seller's email back to him in the comment's text to confirm it.", "intent": "Find me the seller's email if the item depicts Elvis Presley's performance without the audience shown, else leave a comment with the title \"Question\" and text \"Do you have one without the audience?\". Also, repeat the seller's email back to him in the comment's text to confirm it.", "image": null, "instantiation_dict": {}, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=79640", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Question by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=79640", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Do you have one without the audience?", "joo-won_hernandez19@example.com" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "The phone number is spelled wrongly in the item's description. Expectation is that it should be corrected by the agent performing the task.", "intent_template_id": 69 }, { "sites": [ "classifieds" ], "task_id": 214, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the most expensive phone with a theme matching that of the image.", "image": "https://images.pexels.com/photos/3526022/pexels-photo-3526022.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2", "instantiation_dict": { "attribute": "most expensive", "item": "phone", "condition": "with a theme matching that of the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=36313", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 215, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the most expensive camera that is for sale (as confirmed in the listing description).", "image": null, "instantiation_dict": { "attribute": "most expensive", "item": "camera", "condition": "that is for sale (as confirmed in the listing description)" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66657", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 216, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the cheapest oval-shaped table that is between $420 and $430.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "oval-shaped table", "condition": "that is between $420 and $430" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66046", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 217, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} from the classifieds site that {{condition}}.{{action}}", "intent": "Find me the most recent listing offering a book titled \"Captain's Logs: The Complete Trek Voyages\" from the classifieds site that is priced between $170 and $200. Leave a comment on the listing with the title \"Interested in buying the book\" and text \"Is the Captain's Log book still available?\".", "image": null, "instantiation_dict": { "attribute": "most recent listing offering", "item": "a book titled \"Captain's Logs: The Complete Trek Voyages\"", "condition": "is priced between $170 and $200", "action": " Leave a comment on the listing with the title \"Interested in buying the book\" and text \"Is the Captain's Log book still available?\"." }, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=27617", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interested in buying the book by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=27617", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Is the Captain's Log book still available?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 68 }, { "sites": [ "classifieds" ], "task_id": 218, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the most expensive collectible historical USA coin that is under $2000.", "image": null, "instantiation_dict": { "attribute": "most expensive", "item": "collectible historical USA coin", "condition": "that is under $2000" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=82531", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "It only says it is a USA coin in the image itself.", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 219, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}}.", "intent": "Show me the cheapest coat hanger between $15 and $20 that can stand on its own.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "coat hanger", "condition": "between $15 and $20 that can stand on its own" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=34863 |OR| __CLASSIFIEDS__/index.php?page=item&id=27330 |OR| __CLASSIFIEDS__/index.php?page=item&id=62449", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 63 }, { "sites": [ "classifieds" ], "task_id": 220, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__", "geolocation": null, "intent_template": "Find the {{item}} that {{condition}}.", "intent": "Find the cheapest red-colored microwave that is $10 and under and still works.", "image": null, "instantiation_dict": { "item": "cheapest red-colored microwave", "condition": "is $10 and under and still works" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "fuzzy_match": "N/A" }, "reference_url": "", "program_html": [], "string_note": "The only red microwave doesn't turn on" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "The red microwave in question is __CLASSIFIEDS__/index.php?page=item&id=47242", "intent_template_id": 23 }, { "sites": [ "classifieds" ], "task_id": 221, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=48575", "geolocation": null, "intent_template": "How many {{attribute}} are {{listing detail}}?", "intent": "How many bowls are there in this set of bowls?", "image": null, "instantiation_dict": { "attribute": "bowls", "listing detail": "there in this set of bowls" }, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "6" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 37 }, { "sites": [ "classifieds" ], "task_id": 222, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=34501", "geolocation": null, "intent_template": "Based on the measuring tape in the image, is the stated diameter of this item correct?", "intent": "Based on the measuring tape in the image, is the stated diameter of this item correct?", "image": null, "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "yes" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 70 }, { "sites": [ "classifieds" ], "task_id": 223, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=13215", "geolocation": null, "intent_template": "If there is a Santa Claus card or envelope in the listing, help me to comment with the title \"Other cards?\" and with text asking \"Are there any other Santa Claus themed envelopes?\"", "intent": "If there is a Santa Claus card or envelope in the listing, help me to comment with the title \"Other cards?\" and with text asking \"Are there any other Santa Claus themed envelopes?\"", "image": null, "instantiation_dict": {}, "require_reset": true, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=12085", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Other cards? by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=12085", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Are there any other Santa Claus themed envelopes?" ] } } ] }, "reasoning_difficulty": "medium", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 71 }, { "sites": [ "classifieds" ], "task_id": 224, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__/dawn-ultra-dishwashing-liquid-dish-soap-4x19-4-fl-oz-non-scratch-sponge-2-count-original-scent.html", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the cheapest wall rack between $30-40 from the classifieds site that is in the shape of the animal in the item's image on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "wall rack between $30-40", "description": "that is in the shape of the animal in the item's image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=70647", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 225, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__SHOPPING__/womens-hoodies-clearance-tie-die-printed-hoodies-sweatshirt-long-sleeve-drawstring-pullover-tops-hooded-shirts.html |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the most expensive ring from the classifieds site with the same shape as the shape on the right side of the hoodie shown on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "most expensive", "item": "ring", "description": "with the same shape as the shape on the right side of the hoodie shown" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=66601", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 226, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__/5pcs-30ml-cute-mini-portable-gel-silicone-hand-bottle-cartoon-empty-spray-bottle-cosmetic-container-refillable-bottle-hand-soap-bottle.html", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the cheapest figurine from the classifieds site that is of the third animal from the left shown in the product image on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "figurine", "description": "that is of the third animal from the left shown in the product image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=8964", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 227, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__/storage-basket-cube-forest-animal-fox-deer-flower-large-collapsible-toys-storage-box-bin-laundry-organizer-for-closet-shelf-nursery-kids-bedroom-15x11x9-5-in-1-pack.html", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} from the classifieds site {{description}} on OneStopMarket.", "intent": "Show me the cheapest clock from the classifieds site with the green animal at the top of the image on OneStopMarket.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "clock", "description": "with the green animal at the top of the image" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=71222", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 62 }, { "sites": [ "classifieds" ], "task_id": 228, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__ |AND| __SHOPPING__/posterazzi-pdxjp6549small-live-photo-print-12-x-24-multi.html", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} from the classifieds site that {{condition}}.{{action}}", "intent": "Find me the most recent live plant listing from the classifieds site that is the same plant type as the plant shown in the image on OneStopMarket. Then, if there are at least 3 of those plants below 9ft tall shown in the product listing image, comment with the title \"Interested\" and text \"I want 3\".", "image": null, "instantiation_dict": { "attribute": "most recent", "item": "live plant listing", "condition": "is the same plant type as the plant shown in the image on OneStopMarket", "action": " Then, if there are at least 3 of those plants below 9ft tall shown in the product listing image, comment with the title \"Interested\" and text \"I want 3\"." }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=58212", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Interested by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=58212", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "I want 3" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 68 }, { "sites": [ "classifieds" ], "task_id": 229, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=47824 |AND| __CLASSIFIEDS__/index.php?page=item&id=23338 |AND| __CLASSIFIEDS__/index.php?page=item&id=42770", "geolocation": null, "intent_template": "Out of the three tabs, ignoring color, which two {{item}} look the most alike in terms of design? Comment on those two listings, with the title \"Buying\" and text \"Can I get one?\".", "intent": "Out of the three tabs, ignoring color, which two rings look the most alike in terms of design? Comment on those two listings, with the title \"Buying\" and text \"Can I get one?\".", "image": null, "instantiation_dict": { "item": "rings" }, "require_reset": false, "eval": { "eval_types": [ "program_html" ], "reference_answers": null, "reference_url": "", "program_html": [ { "url": "__CLASSIFIEDS__/index.php?page=item&id=47824", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Buying by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=47824", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Can I get one?" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=42770", "locator": "func:get_query_text(__page__, '.comments_list h3')", "required_contents": { "must_include": [ "Buying by Blake Sullivan" ] } }, { "url": "__CLASSIFIEDS__/index.php?page=item&id=42770", "locator": "func:get_query_text(__page__, '.comments_list')", "required_contents": { "must_include": [ "Can I get one?" ] } } ] }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 72 }, { "sites": [ "classifieds" ], "task_id": 230, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__SHOPPING__/sudugo-smart-watch-2021-ver-watches-for-women-1-69-touch-screen-fitness-tracker-with-heart-rate-and-sleep-monitor-ip67-waterproof-smartwatch-calorie-pedometer-activity-tracker-for-android-ios-pink.html |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} from the classifieds site that {{condition}}.{{action}}", "intent": "Find me the cheapest watch in the Jewelry category from the classifieds site that roughly matches the time shown on the first image of this watch on OneStopMarket, up to an error of plus minus 10 minutes.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "watch in the Jewelry category", "condition": "roughly matches the time shown on the first image of this watch on OneStopMarket, up to an error of plus minus 10 minutes", "action": "" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=56727", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 68 }, { "sites": [ "classifieds" ], "task_id": 231, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__REDDIT__/f/Maine/99914/new-tourist-trap |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Show me the {{attribute}} {{item}} {{condition}} from the classifieds site that looks the most similar to the flower in the image on Reddit. It doesn't have to be the same color.", "intent": "Show me the cheapest flower in the Farm + garden category under $10 from the classifieds site that looks the most similar to the flower in the image on Reddit. It doesn't have to be the same color.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "flower in the Farm + garden category", "condition": "under $10" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=69345 |OR| __CLASSIFIEDS__/index.php?page=item&id=10003", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 73 }, { "sites": [ "classifieds" ], "task_id": 232, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__REDDIT__/f/consoles/124577/name-a-better-console-you-can-t |AND| __CLASSIFIEDS__", "geolocation": null, "intent_template": "Find me the {{attribute}} {{item}} from the classifieds site that {{condition}}.{{action}}", "intent": "Find me the cheapest controller from the classifieds site that is meant for the console in the image on the Reddit tab.", "image": null, "instantiation_dict": { "attribute": "cheapest", "item": "controller", "condition": "is meant for the console in the image on the Reddit tab", "action": "" }, "require_reset": false, "eval": { "eval_types": [ "url_match" ], "reference_answers": null, "reference_url": "__CLASSIFIEDS__/index.php?page=item&id=33011", "program_html": [], "url_note": "EXACT" }, "reasoning_difficulty": "hard", "visual_difficulty": "hard", "overall_difficulty": "hard", "comments": "", "intent_template_id": 68 }, { "sites": [ "classifieds" ], "task_id": 233, "require_login": true, "storage_state": "./.auth/classifieds_state.json", "start_url": "__CLASSIFIEDS__/index.php?page=item&id=28914 |AND| __REDDIT__/f/springfieldMO/48540/never", "geolocation": null, "intent_template": "Are the characters in the image on Reddit shown in the listing on the classifieds site? If they are, name me the movie that they appear in.", "intent": "Are the characters in the image on Reddit shown in the listing on the classifieds site? If they are, name me the movie that they appear in.", "image": null, "instantiation_dict": {}, "require_reset": false, "eval": { "eval_types": [ "string_match" ], "reference_answers": { "must_include": [ "lion king" ] }, "reference_url": "", "program_html": [], "string_note": "" }, "reasoning_difficulty": "easy", "visual_difficulty": "easy", "overall_difficulty": "easy", "comments": "", "intent_template_id": 74 } ]