yuyr/AgentOccam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
4b8dbfd9c5
AgentOccam/config_files/test.raw.json
Yao Liu 4b8dbfd9c5 init
2025-01-22 11:32:35 -08:00

29624 lines
862 KiB
JSON
Raw Blame History

[
{
"sites": [
"shopping_admin"
],
"task_id": 0,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 1,
"year": 2022
},
"intent": "What is the top-1 best-selling product in 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Quest Lumaflex\u2122 Band"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 1,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling brand in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Quarter 1 2022"
},
"intent": "What is the top-1 best-selling brand in Quarter 1 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Sprite"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sprite"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 2,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Quarter 1 2022"
},
"intent": "What is the top-1 best-selling product type in Quarter 1 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Yoga ball"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yoga ball"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 3,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 2,
"year": 2022
},
"intent": "What are the top-2 best-selling product in 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Quest Lumaflex\u2122 Band",
"Sprite Stasis Ball 65 cm"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band, Sprite Stasis Ball 65 cm"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 4,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{period}}",
"instantiation_dict": {
"n": 3,
"period": "Jan 2023"
},
"intent": "What are the top-3 best-selling product in Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Impulse Duffle",
"Overnight Duffle",
"Hawkeye Yoga Short-32-Blue"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Impulse Duffle, Overnight Duffle, Hawkeye Yoga Short-32-Blue"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 5,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Jan 2023"
},
"intent": "What is the top-1 best-selling product type in Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Duffle"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Duffle"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 6,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 5,
"year": 2023
},
"intent": "What are the top-5 best-selling product in 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sprite Yoga Strap 6 foot",
"Overnight Duffle",
"Ida Workout Parachute Pant-29-Purple",
"Hawkeye Yoga Short-32-Blue",
"Sprite Stasis Ball 65 cm"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sprite Yoga Strap 6 foot, Overnight Duffle, Ida Workout Parachute Pant-29-Purple, Hawkeye Yoga Short-32-Blue, Sprite Stasis Ball 65 cm"
},
"intent_template_id": 279
},
{
"sites": [
"map"
],
"task_id": 7,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Mellon University",
"radius": "50 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 50 km to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 8,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Mellon University",
"radius": "5 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 5 km to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"There is no airport within 5 km of Carnegie Mellon University"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "There is no airport within 5 km of Carnegie Mellon University"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 9,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Art Museum",
"radius": "30 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 30 km to Carnegie Art Museum",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 10,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "US international airports",
"start": "Niagara Falls",
"radius": "60 km"
},
"intent": "Tell me the full address of all US international airports that are within a driving distance of 60 km to Niagara Falls",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States",
"Buffalo-Niagara International Airport, Holtz Drive, Town of Cheektowaga, Erie County, New York, 14225, United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States Buffalo-Niagara International Airport, South Youngs Road, Town of Cheektowaga, Erie County, New York, 14221, United States"
},
"intent_template_id": 79
},
{
"sites": [
"shopping_admin"
],
"task_id": 11,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "disappointed"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"disappointed\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"6"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "6"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 12,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "satisfied"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"satisfied\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 13,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "decent"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"decent\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 14,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "not useful"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"not useful\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 15,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "best"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"best\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"map"
],
"task_id": 16,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "5000 Fifth Avenue, Pittsburgh",
"end": "UPMC family health center"
},
"intent": "Compare the time for walking and driving route from 5000 Fifth Avenue, Pittsburgh to UPMC family health center",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 2min; walking: 16min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Driving: 2min. Walking: 16min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 17,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "AMC Waterfront",
"end": "Carnegie Mellon University"
},
"intent": "Compare the time for walking and driving route from AMC Waterfront to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 13min; walking: 1h 35min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 13min, walking: 1h 35min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 18,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "AMC Waterfront",
"end": "Univ of Pittsburgh"
},
"intent": "Compare the time for walking and driving route from AMC Waterfront to Univ of Pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 15min; walking: 1h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 15min, walking: 1h 47min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 19,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "Carnegie Science Center",
"end": "Carnegie Mellon University"
},
"intent": "Compare the time for walking and driving route from Carnegie Science Center to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 12min; walking: 1h 44min."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 12min, walking: 1h 44min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 20,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the difference in time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "Randyland",
"end": "Carnegie Mellon University"
},
"intent": "Compare the difference in time for walking and driving route from Randyland to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 13min; walking: 1h 45min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 13min, walking: 1h 45min."
},
"intent_template_id": 73
},
{
"sites": [
"shopping"
],
"task_id": 21,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/6s-wireless-headphones-over-ear-noise-canceling-hi-fi-bass-foldable-stereo-wireless-kid-headsets-earbuds-with-built-in-mic-micro-sd-tf-fm-for-iphone-samsung-ipad-pc-black-gold.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "ear cups being small"
},
"intent": "List out reviewers, if exist, who mention about ear cups being small",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Joseph Brzezinski",
"Catso",
"Dibbins",
"Anglebert Dinkherhump",
"Michelle Davis"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Joseph Brzezinski, Catso, Dibbins, Anglebert Dinkherhump, Michelle Davis"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 22,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/fujifilm-finepix-z200fd-10mp-digital-camera-with-5x-optical-dual-image-stabilized-zoom-black.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "under water photo"
},
"intent": "List out reviewers, if exist, who mention about under water photo",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no review about under water photo",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 23,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/3-pack-samsung-galaxy-s6-screen-protector-nearpow-tempered-glass-screen-protector-with-9h-hardness-crystal-clear-easy-bubble-free-installation-scratch-resist.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "good fingerprint resistant"
},
"intent": "List out reviewers, if exist, who mention about good fingerprint resistant",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rachel",
"T. Gannon"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rachel, T. Gannon, "
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 24,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/haflinger-men-s-wool-felt-open-back-slippers-beige-550-peat-us-7.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "price being unfair"
},
"intent": "List out reviewers, if exist, who mention about price being unfair",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no reivew about price being unfair",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 25,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "average print quality"
},
"intent": "List out reviewers, if exist, who mention about average print quality",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Goldfish",
"Roxanne Brandon Coffey"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "GoldfishGoldfish, Roxanne Brandon Coffey"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 26,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "complain of the customer service"
},
"intent": "List out reviewers, if exist, who mention about complain of the customer service",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Bob in Vegas",
"RemyR"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Bob in Vegas, RemyRRemyR"
},
"intent_template_id": 222
},
{
"sites": [
"reddit"
],
"task_id": 27,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "Showerthoughts"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Showerthoughts forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 28,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "Worcester"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Worcester forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 29,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "DIY"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the DIY forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 30,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "space"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the space forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 31,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "photoshopbattles"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the photoshopbattles forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"map"
],
"task_id": 32,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"information": "the walking distance",
"target1": "Hilton hotel",
"target2": "the nearest supermarket own by a local company"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the walking distance to the nearest supermarket own by a local company from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"DoubleTree by Hilton Hotel Pittsburgh Airport",
"2.0km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 2.0km"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 33,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hilton hotel",
"information": "the shortest walking distance",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the shortest walking distance to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"DoubleTree by Hilton Hotel Pittsburgh Airport",
"1.4km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 1.4km"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 34,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hyatt hotel",
"information": "the shortest walking time",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the shortest walking time to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport"
],
"fuzzy_match": [
"Time: 3h 30min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport\n3:30"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 35,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hyatt hotel",
"information": "the minimal driving time",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the minimal driving time to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport"
],
"fuzzy_match": [
"Time: 15min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Time: 15min"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 36,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "social security administration",
"location": "Carnegie Mellon University"
},
"intent": "Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 37,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "police station",
"location": "gates building at CMU"
},
"intent": "Check if the police station in pittsburgh can be reached in one hour by car from gates building at CMU",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 38,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "duquesne university",
"location": "pittsburgh airport"
},
"intent": "Check if the duquesne university in pittsburgh can be reached in one hour by car from pittsburgh airport",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 39,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "walmart",
"location": "5600 fifth avenue"
},
"intent": "Check if the walmart in pittsburgh can be reached in one hour by car from 5600 fifth avenue",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 40,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "amc theatre",
"location": "hobart street"
},
"intent": "Check if the amc theatre in pittsburgh can be reached in one hour by car from hobart street",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"shopping_admin"
],
"task_id": 41,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "1"
},
"intent": "List the top 1 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "hollister"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister"
},
"intent_template_id": 285
},
{
"sites": [
"shopping_admin"
],
"task_id": 42,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "2"
},
"intent": "List the top 2 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"hollister",
"Joust Bag"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister, Joust Bag"
},
"intent_template_id": 285
},
{
"sites": [
"shopping_admin"
],
"task_id": 43,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "3"
},
"intent": "List the top 3 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"hollister",
"Joust Bag",
"Antonia Racer Tank"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister, Joust Bag, Antonia Race Tank"
},
"intent_template_id": 285
},
{
"sites": [
"gitlab"
],
"task_id": 44,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Check out my todos",
"instantiation_dict": {},
"intent": "Check out my todos",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/todos",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 303
},
{
"sites": [
"gitlab"
],
"task_id": 45,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "Check out the most recent open issues",
"instantiation_dict": {},
"intent": "Check out the most recent open issues",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?sort=created_asc&state=opened",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 300
},
{
"sites": [
"gitlab"
],
"task_id": 46,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Check out the most recent open issues",
"instantiation_dict": {},
"intent": "Check out the most recent open issues",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/issues/?sort=created_date&state=opened",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 300
},
{
"sites": [
"shopping"
],
"task_id": 47,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"0 order",
"$0 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0 order, $0 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 48,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past three days"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past three days, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"0 order",
"$0 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0 order, $0 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 49,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past four month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past four month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3 orders",
"$845.49 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3 orders, $845.49 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 50,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past year"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past year, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"24 orders",
"$6560.69 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "24 orders, $6560.69 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 51,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past six month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past six month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"12 orders",
"$1603.69 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "12 orders, $1603.69 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"map"
],
"task_id": 52,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Mellon University",
"end": "starbucks on Craig Street"
},
"intent": "How long does it take to walk from Carnegie Mellon University to starbucks on Craig Street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"7 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "7 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 53,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Univ of Pittsburgh",
"end": "starbucks on Craig Street"
},
"intent": "How long does it take to walk from Univ of Pittsburgh to starbucks on Craig Street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"18 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "18 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 54,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Mellon University",
"end": "Univ of Pittsburgh"
},
"intent": "How long does it take to walk from Carnegie Mellon University to Univ of Pittsburgh?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"25 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "25 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 55,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "the starbuck near CMU",
"end": "Chatham university"
},
"intent": "How long does it take to walk from the starbuck near CMU to Chatham university?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"30 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "30 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 56,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Museum of Art",
"end": "a library at CMU"
},
"intent": "How long does it take to walk from Carnegie Museum of Art to a library at CMU?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"11 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "11 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 57,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "university center at Carnegie Mellon University"
},
"intent": "Tell me the closest restaurant(s) to university center at Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"El Gallo de Oro",
"Back Bar Grill",
"Grano",
"Beefsteak",
"Nourish",
"Schatz Dining Room",
"Au Bon Pain"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "El Gallo de Oro, Back Bar Grill, Grano, Beefsteak, Nourish, Schatz Dining Room, Au Bon Pain"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 58,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "cafe",
"place2": "CMU Hunt library"
},
"intent": "Tell me the closest cafe(s) to CMU Hunt library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"De Fer Coffee & Tea"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "De Fer Coffee & Tea"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 59,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Hunt library"
},
"intent": "Tell me the closest restaurant(s) to CMU Hunt library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The exchange"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The exchange"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 60,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Posner Hall"
},
"intent": "Tell me the closest restaurant(s) to CMU Posner Hall",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The exchange"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The exchange"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 61,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Sorrells Library"
},
"intent": "Tell me the closest restaurant(s) to CMU Sorrells Library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"La Prima Espresso"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "La Prima Espresso"
},
"intent_template_id": 69
},
{
"sites": [
"shopping_admin"
],
"task_id": 62,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "most"
},
"intent": "Which customer has completed the most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jane Smith"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jane Smith"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 63,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer(s) has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "second most"
},
"intent": "Which customer(s) has completed the second most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Adam Garcia",
"Michael Nguyen",
"Sarah Miller"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Adam Garcia, Michael Nguyen, Sarah Miller"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 64,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has placed {{number}} orders in the entire history?",
"instantiation_dict": {
"number": "2"
},
"intent": "Which customer has placed 2 orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Kim",
"Lisa Green",
"Julia Williams",
"Brian Smith",
"Alexander Thomas"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Kim, Lisa Green, Julia Williams, Brian Smith, Alexander Thomas"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 65,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "fifth most"
},
"intent": "Which customer has completed the fifth most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jane Doe"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jane Doe"
},
"intent_template_id": 276
},
{
"sites": [
"reddit"
],
"task_id": 66,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the post URLs that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the post URLs that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"http://www.reddit.com/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol",
"http://www.reddit.com/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "__REDDIT__/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol, __REDDIT__/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 67,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the book names from posts that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the book names from posts that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"A Christmas Carol",
"The Hobbit"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "A Christmas Carol, The Hobbit"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 68,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the author name and the book name from posts that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the author name and the book name from posts that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"A Christmas Carol",
"Levar Burton",
"The Hobbit",
"J. R. R. Tolkien"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "A Christmas Carol by Levar Burton: , The Hobbit by J. R. R. Tolkien"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 69,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "is there any post talks about supporting local book stores? If so, tell me the organizations involved"
},
"intent": "Among the top 10 post in \"books\" forum, is there any post talks about supporting local book stores? If so, tell me the organizations involved",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"bookshop.org"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "bookshop.org"
},
"intent_template_id": 17
},
{
"sites": [
"map"
],
"task_id": 70,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Carnegie Mellon University"
},
"intent": "What is the zip code of Carnegie Mellon University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "15213"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "15213"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 71,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Chatham University"
},
"intent": "What is the zip code of Chatham University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "15232"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "15232"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 72,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Yale University"
},
"intent": "What is the zip code of Yale University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "06516"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "06516"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 73,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Columbia University"
},
"intent": "What is the zip code of Columbia University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "10027"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "10027"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 74,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Carnegie Mellon University",
"apple store shadyside",
"starbucks on craig street"
]
},
"intent": "Given the following locations, ['Carnegie Mellon University', 'apple store shadyside', 'starbucks on craig street'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The order is Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
},
"intent_template_id": 65
},
{
"sites": [
"map"
],
"task_id": 75,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Massachusetts Institute of Technology",
"Harvard University",
"Boston Logan International Airport"
]
},
"intent": "Given the following locations, ['Massachusetts Institute of Technology', 'Harvard University', 'Boston Logan International Airport'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The order is Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
},
"intent_template_id": 65
},
{
"sites": [
"map"
],
"task_id": 76,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Princeton University",
"Yale University",
"Harvard University"
]
},
"intent": "Given the following locations, ['Princeton University', 'Yale University', 'Harvard University'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The order is Princeton University, Yale University, Harvard University"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Princeton University, Yale University, Harvard University"
},
"intent_template_id": 65
},
{
"sites": [
"shopping_admin"
],
"task_id": 77,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Pending"
},
"intent": "What is the total count of Pending reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5"
},
"intent_template_id": 277
},
{
"sites": [
"shopping_admin"
],
"task_id": 78,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Approved"
},
"intent": "What is the total count of Approved reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"346"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "346"
},
"intent_template_id": 277
},
{
"sites": [
"shopping_admin"
],
"task_id": 79,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Not Approved"
},
"intent": "What is the total count of Not Approved reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 277
},
{
"sites": [
"map"
],
"task_id": 80,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Carnegie Mellon University",
"place_B": "Starbucks on Craig Street",
"place_C": "Pittsburgh International Airport"
},
"intent": "What is the duration required to first walk from Carnegie Mellon University to Starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"38 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "38 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 81,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Univ of Pittsburgh",
"place_B": "starbucks on Craig Street",
"place_C": "Pittsburgh International Airport"
},
"intent": "What is the duration required to first walk from Univ of Pittsburgh to starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"49 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "49 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 82,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Massachusetts Institute of Technology",
"place_B": "Harvard University",
"place_C": "Boston Logan International Airport"
},
"intent": "What is the duration required to first walk from Massachusetts Institute of Technology to Harvard University, and then drive to Boston Logan International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"63 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "63 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 83,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Carnegie Mellon University",
"place_B": "apple store shadyside",
"place_C": "starbucks on craig street"
},
"intent": "What is the duration required to first walk from Carnegie Mellon University to apple store shadyside, and then drive to starbucks on craig street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"22 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "22 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 84,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "DoubleTree by Hilton New York Downtown",
"place": "Keens Steakhouse"
},
"intent": "From my stay at DoubleTree by Hilton New York Downtown, what's the estimated driving time to reach Keens Steakhouse?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"14 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "14 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 85,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "La Quinta Inn near the airport",
"place": "Carnegie Mellon University"
},
"intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Carnegie Mellon University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"30 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "30 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 86,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "La Quinta Inn near the airport",
"place": "Upitt"
},
"intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Upitt?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"29 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "29 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 87,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "red roof inn",
"place": "Pittsburgh science museum"
},
"intent": "From my stay at red roof inn, what's the estimated driving time to reach Pittsburgh science museum?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"20 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "20 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 88,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "Homewood Suites Southpointe",
"place": "PPG Paints Arena"
},
"intent": "From my stay at Homewood Suites Southpointe, what's the estimated driving time to reach PPG Paints Arena?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"34 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "34 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 89,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Connecticut"
},
"intent": "Which US states border Connecticut?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rhode Island",
"Massachusetts",
"New York"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rhode Island, Massachusetts, New York"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 90,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Pennsylvania"
},
"intent": "Which US states border Pennsylvania?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Ohio",
"Maryland",
"New York",
"New Jersey",
"Delaware",
"West Virginia"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Ohio, Maryland, New York, New Jersey, Delaware, West Virginia"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 91,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Massachusetts"
},
"intent": "Which US states border Massachusetts?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rhode Island",
"Connecticut",
"New York",
"New Hampshire",
"Vermont"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rhode Island, Connecticut, New York, New Hampshire, Vermont"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 92,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Vermont"
},
"intent": "Which US states border Vermont?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"New York",
"New Hampshire",
"Massachusetts"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "New York, New Hampshire, Massachusetts"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 93,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "New Hampshire"
},
"intent": "Which US states border New Hampshire?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Massachusetts",
"Vermont",
"Maine"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Massachusetts, Vermont, Maine"
},
"intent_template_id": 67
},
{
"sites": [
"shopping_admin"
],
"task_id": 94,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Telll me the grand total of invoice {{id}}.",
"instantiation_dict": {
"id": "000000001"
},
"intent": "Telll me the grand total of invoice 000000001.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"36.39"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$36.39"
},
"intent_template_id": 274
},
{
"sites": [
"shopping_admin"
],
"task_id": 95,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Telll me the grand total of invoice {{id}}.",
"instantiation_dict": {
"id": "000000002"
},
"intent": "Telll me the grand total of invoice 000000002.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"39.64"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$39.64"
},
"intent_template_id": 274
},
{
"sites": [
"shopping"
],
"task_id": 96,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the status of my latest order and when will it arrive",
"instantiation_dict": {},
"intent": "Tell me the status of my latest order and when will it arrive",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The last order was canceled. It will never arrive."
]
},
"reference_url": "",
"program_html": [],
"reference_answer_raw_annotation": "The last order was canceled. It will never arrive.",
"string_note": ""
},
"intent_template_id": 193
},
{
"sites": [
"map",
"wikipedia"
],
"task_id": 97,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
"instantiation_dict": {},
"intent": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"914km |OR| 914 km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "914 km"
},
"intent_template_id": 120
},
{
"sites": [
"map"
],
"task_id": 98,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "tea cafe",
"start": "University of Pittsburgh"
},
"intent": "Where is the nearest tea cafe to University of Pittsburgh, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Fuku Tea",
"3716",
"Forbes Avenue",
"Central Oakland",
"Pittsburgh",
"653m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Fuku Tea, 3716, Forbes Avenue, Oakland, Central Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n653m"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 99,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "Five Guys",
"start": "5700 Penn Ave"
},
"intent": "Where is the nearest Five Guys to 5700 Penn Ave, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Five Guys",
"117",
"South Bouquet Street",
"North Oakland",
"Pittsburgh",
"4.0km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Five Guys, 117, South Bouquet Street, Oakland, North Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n4.0km"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 100,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "Starbucks",
"start": "Carnegie Mellon"
},
"intent": "Where is the nearest Starbucks to Carnegie Mellon, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Starbucks",
"417",
"South Craig Street",
"Bellefield",
"Pittsburgh",
"557m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Starbucks, 417, South Craig Street, Bellefield, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n557m"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 101,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "In-N-Out",
"start": "Upitts"
},
"intent": "Where is the nearest In-N-Out to Upitts, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no In-N-Out near University of Pittsburgh",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 66
},
{
"sites": [
"gitlab"
],
"task_id": 102,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "help needed",
"repo": "a11yproject/a11yproject.com"
},
"intent": "Display the list of issues in the a11yproject/a11yproject.com repository that have labels related to help needed",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/?label_name%5B%5D=help%20wanted",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 103,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "questions",
"repo": "kkroening/ffmpeg-python"
},
"intent": "Display the list of issues in the kkroening/ffmpeg-python repository that have labels related to questions",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/kkroening/ffmpeg-python/-/issues/?label_name%5B%5D=question",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 104,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "flaky-test",
"repo": "keycloak/keycloak"
},
"intent": "Display the list of issues in the keycloak/keycloak repository that have labels related to flaky-test",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/keycloak/keycloak/-/issues/?label_name%5B%5D=flaky-test",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 105,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "OpenAPI Generator CLI",
"repo": "OpenAPITools/openapi-generator"
},
"intent": "Display the list of issues in the OpenAPITools/openapi-generator repository that have labels related to OpenAPI Generator CLI",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/OpenAPITools/openapi-generator/-/issues/?label_name%5B%5D=OpenAPI%20Generator%20CLI",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 106,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "BUG",
"repo": "umano/AndroidSlidingUpPanel"
},
"intent": "Display the list of issues in the umano/AndroidSlidingUpPanel repository that have labels related to BUG",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/umano/AndroidSlidingUpPanel/-/issues/?label_name%5B%5D=BUG",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"shopping_admin"
],
"task_id": 107,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from May to December 2022"
},
"intent": "Presents the monthly count of successful orders from May to December 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"October: 4 orders",
"November: 5 orders",
"December: 10 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 108,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "01/2023-05/2023"
},
"intent": "Presents the monthly count of successful orders 01/2023-05/2023 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 12 orders",
"Feburary: 7 orders",
"March: 5 orders",
"April: 9 orders",
"May: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 12 orders Febulary: 7 orders March: 5 orders Apirl: 9 orders May: 5 orders"
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 109,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Jan to December 2022"
},
"intent": "Presents the monthly count of successful orders from Jan to December 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 11 orders",
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders",
"December: 10 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 110,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Jan to Nov 2022"
},
"intent": "Presents the monthly count of successful orders from Jan to Nov 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 11 orders",
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 111,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Feb to Nov 2022"
},
"intent": "Presents the monthly count of successful orders from Feb to Nov 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 112,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Circe fleece"
},
"intent": "Show me the customers who have expressed dissatisfaction with Circe fleece?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Hannah Lim"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hannah Lim"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 113,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Olivia zip jacket"
},
"intent": "Show me the customers who have expressed dissatisfaction with Olivia zip jacket?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Emma Lopez",
"Seam Miller"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Emma Lopez, Seam Miller"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 114,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Antonia racer tank"
},
"intent": "Show me the customers who have expressed dissatisfaction with Antonia racer tank?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Shaunte",
"Merrie"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shaunte, Merrie"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 115,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}",
"instantiation_dict": {
"product": "Chloe tank"
},
"intent": "Show me the name of the customers who have expressed dissatisfaction with Chloe tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no negative review for Chloe tank",
"reference_answer_raw_annotation": ""
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 116,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "tanks products"
},
"intent": "Show me the name of the customers who have expressed dissatisfaction with tanks products?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Alexander",
"Carma",
"Dominic",
"Merrie",
"Monroe",
"Scotty",
"Shaunte",
"Teofila",
"Valorie"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Alexander, Carma, Dominic, Merrie, Monroe, Scotty, Shaunte, Teofila, Valorie"
},
"intent_template_id": 245
},
{
"sites": [
"shopping"
],
"task_id": 117,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the date when I made my first purchase on this site?",
"instantiation_dict": {},
"intent": "What is the date when I made my first purchase on this site?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3/2/22"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3/2/22"
},
"intent_template_id": 161
},
{
"sites": [
"shopping"
],
"task_id": 118,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have jaw bruxism problem, show me something that could alleviate the problem.",
"instantiation_dict": {},
"intent": "I have jaw bruxism problem, show me something that could alleviate the problem.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "",
"required_contents": {
"must_include": [
"jaw bruxism",
"mouth guard"
]
}
}
]
},
"intent_template_id": 151
},
{
"sites": [
"shopping_admin"
],
"task_id": 119,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "Tell me the reasons why customers like Antonia Racer Tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Its color and style is good"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Its color and style is good"
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 120,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Ana Running Short"
},
"intent": "Tell me the reasons why customers like Ana Running Short",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"It is comfortable"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "It is comfortable"
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 121,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Circe hooded fleece"
},
"intent": "Tell me the reasons why customers like Circe hooded fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Warm and comfortable. True to size."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Warm and comfortable. True to size."
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 122,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Olivia zip jacket"
},
"intent": "Tell me the reasons why customers like Olivia zip jacket",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Lightweight, comfortable and stylish. Good design and details."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lightweight, comfortable, and stylish. Good design and details."
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 123,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Circe's products"
},
"intent": "Tell me the reasons why customers like Circe's products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Warm and comfortable. True to size."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Warm and comfortable. True to size."
},
"intent_template_id": 250
},
{
"sites": [
"shopping"
],
"task_id": 124,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "wireless earphone"
},
"intent": "What is the price range of wireless earphone in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0.14",
"745.00"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$0.14 - $745.00"
},
"intent_template_id": 159
},
{
"sites": [
"shopping"
],
"task_id": 125,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "teeth grinding mouth guard"
},
"intent": "What is the price range of teeth grinding mouth guard in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1.46",
"85"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$1.46 - $85"
},
"intent_template_id": 159
},
{
"sites": [
"shopping"
],
"task_id": 126,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "Canon photo printer"
},
"intent": "What is the price range of Canon photo printer in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2.56",
"649.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$2.56 - $649.99"
},
"intent_template_id": 159
},
{
"sites": [
"shopping_admin"
],
"task_id": 127,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What brands appear most frequently among the top search terms?",
"instantiation_dict": {},
"intent": "What brands appear most frequently among the top search terms?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hollister",
"Joust",
"Antonia"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hollister, Joust, Antonia"
},
"intent_template_id": 1001
},
{
"sites": [
"shopping_admin"
],
"task_id": 128,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "2"
},
"intent": "What's the total number of items sold in the most recent 2 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 129,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "4"
},
"intent": "What's the total number of items sold in the most recent 4 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 130,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "5"
},
"intent": "What's the total number of items sold in the most recent 5 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"18"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "18"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 131,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "7"
},
"intent": "What's the total number of items sold in the most recent 7 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"25"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "25"
},
"intent_template_id": 1002
},
{
"sites": [
"gitlab"
],
"task_id": 132,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"repo": "a11yproject",
"date": "3/5/2023"
},
"intent": "How many commits did kilian make to a11yproject on 3/5/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 133,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Eric",
"repo": "a11yproject",
"date": "3/2"
},
"intent": "How many commits did Eric make to a11yproject on 3/2?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 134,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"repo": "a11yproject",
"date": "3/1/2023"
},
"intent": "How many commits did kilian make to a11yproject on 3/1/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 135,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Eric and Kilian",
"repo": "a11yproject",
"date": "1/3/2023"
},
"intent": "How many commits did Eric and Kilian make to a11yproject on 1/3/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 136,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Steven Woodson",
"repo": "a11y-webring.club",
"date": "2/6/2023"
},
"intent": "How many commits did Steven Woodson make to a11y-webring.club on 2/6/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5"
},
"intent_template_id": 322
},
{
"sites": [
"map"
],
"task_id": 137,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the city where the Liberty Bell is located",
"city2": "the home city of Pirates"
},
"intent": "What is the estimated driving time between the city where the Liberty Bell is located and the home city of Pirates?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"5h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5h 47min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 138,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the big apple",
"city2": "the city with the most authentic Philly cheesesteaks"
},
"intent": "What is the estimated driving time between the big apple and the city with the most authentic Philly cheesesteaks?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"1h 58min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1h 58min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 139,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the hometown of Joe Biden",
"city2": "Bridgeport"
},
"intent": "What is the estimated driving time between the hometown of Joe Biden and Bridgeport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3h 20min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3h 20min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 140,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the city of Niagara Falls",
"city2": "the city of Yale University"
},
"intent": "What is the estimated driving time between the city of Niagara Falls and the city of Yale University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"8h 33min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "8h 33min"
},
"intent_template_id": 51
},
{
"sites": [
"shopping"
],
"task_id": 141,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "food-related",
"time": "March 2023"
},
"intent": "How much I spent on food-related shopping during March 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"47.41"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$47.41"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 142,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "hair care and hair style",
"time": "Jan 2023"
},
"intent": "How much I spent on hair care and hair style shopping during Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"95.23"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$95.23"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 143,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "home decoration",
"time": "1/29/2023"
},
"intent": "How much I spent on home decoration shopping during 1/29/2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"265.69"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$265.69"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 144,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "food",
"time": "from mid Jan to the end Jan 2023"
},
"intent": "How much I spent on food shopping during from mid Jan to the end Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 145,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "cooking and food",
"time": "March 2022"
},
"intent": "How much I spent on cooking and food shopping during March 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"52.35"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$52.35"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 146,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "size",
"product": "picture frame",
"time": "Sep 2022"
},
"intent": "What is the size configuration of the picture frame I bought Sep 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"16x24"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16x24"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 147,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "size",
"product": "picture frame",
"time": "2022"
},
"intent": "What is the size configuration of the picture frame I bought 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16x24"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16x24"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 148,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "color",
"product": "picture frame",
"time": "Sep 2022"
},
"intent": "What is the color configuration of the picture frame I bought Sep 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Mist"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Mist"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 149,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "color",
"product": "artifical plants",
"time": "Feb 2023"
},
"intent": "What is the color configuration of the artifical plants I bought Feb 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Green-vines"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Green-vines"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 150,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "price",
"product": "fake tree",
"time": "Jan 2023"
},
"intent": "What is the price configuration of the fake tree I bought Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"260.69"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "260.69"
},
"intent_template_id": 155
},
{
"sites": [
"map"
],
"task_id": 151,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "CMU",
"location2": "University of Pittsburgh"
},
"intent": "What is the minimum travel time by car from CMU to University of Pittsburgh?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 152,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "Schenley park",
"location2": "Upitt"
},
"intent": "What is the minimum travel time by car from Schenley park to Upitt?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 153,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "REI",
"location2": "CMU"
},
"intent": "What is the minimum travel time by car from REI to CMU?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"7min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "7min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 154,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "CMU gates building",
"location2": "Schenley park"
},
"intent": "What is the minimum travel time by car from CMU gates building to Schenley park?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 155,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "Animal Rescue League of Pittsburgh",
"location2": "Schenley park"
},
"intent": "What is the minimum travel time by car from Animal Rescue League of Pittsburgh to Schenley park?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"9min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9min"
},
"intent_template_id": 36
},
{
"sites": [
"gitlab"
],
"task_id": 156,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Checkout merge requests assigned to me",
"instantiation_dict": {},
"intent": "Checkout merge requests assigned to me",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/merge_requests?assignee_username=byteblaze",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 290
},
{
"sites": [
"shopping_admin"
],
"task_id": 157,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show all customers",
"instantiation_dict": {},
"intent": "Show all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/customer/index/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 255
},
{
"sites": [
"shopping"
],
"task_id": 158,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 11
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 11 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 159,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 31
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 31 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 160,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 6
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 6 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 161,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 23
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 23 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 162,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 40
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 40 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 163,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"I ordered the 16gb but I only got 14 gigs even though I formatted the card",
"The memory card is kind of slow on games and downloads",
"No original packaging It's used and the previous owners data has not been erased",
"The product is a legit sony hardware that have been owned by someone else before",
"The media could not be loaded",
"I could not format the card so I wasn\u2019t able to use it for my VITA"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "I ordered the 16gb but I only got 14 gigs even though I formatted the card. The memory card is kind of slow on games and downloads. No original packaging It's used and the previous owners data has not been erased. The product is a legit sony hardware that have been owned by someone else before The media could not be loaded. I could not format the card so I wasn\u2019t able to use it for my VITA"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 164,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/mineralogie-all-natural-lip-gloss-ruby-rose.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Dry",
"Uneven color"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DryUneven color"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 165,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/sandgrens-swedish-handmade-wooden-clog-sandal-copenhagen.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The 39 was too small. I am afraid the 40 will be too big",
"I was very sad when the shoe rubbed up against my baby toe",
"I had to return them because I knew in time it would tear up my feet",
"The problem is that the strap is made of some really stiff leather and is painful to my heel",
"The front is also uncomfortably tight",
"The Dansko's were similar (not as bad) and loosened up over time"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The 39 was too small. I am afraid the 40 will be too big. I was very sad when the shoe rubbed up against my baby toe. I had to return them because I knew in time it would tear up my feet. The problem is that the strap is made of some really stiff leather and is painful to my heel. The front is also uncomfortably tight. The Dansko's were similar (not as bad) and loosened up over time."
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 166,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/sensodyne-repair-protect-whitening-toothpaste-with-fluoride-3-4-oz-pack-of-3.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "there is no existing criticism",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 167,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/photosmart-plus-b209-clr-inkjetfb-p-s-c-usb-wrls-1.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The wireless connection works on a whim (about 40% of the time I've owned it)",
"It seems to constantly run out of ink",
"Cartridge prices are less than some printers I've had",
"This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason",
"Scanner is as slow as my first scanner I ever owned in the mid-90's",
"For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The wireless connection works on a whim (about 40% of the time I've owned it). It seems to constantly run out of ink. Cartridge prices are less than some printers I've had, but now I understand why. This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason. Scanner is as slow as my first scanner I ever owned in the mid-90's. For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
},
"intent_template_id": 136
},
{
"sites": [
"gitlab"
],
"task_id": 168,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "more than 100"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got more than 100 stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "No repo found",
"reference_answer_raw_annotation": "No repo found"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 169,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "the most"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got the most stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"a11yproject.com",
"design"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "a11yproject.com, Primer/design"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 170,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "the least"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got the least stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 171,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "less than 5"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got less than 5 stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"a11y-syntax-highlighting",
"a11y-webring.club",
"accessible-html-content-patterns",
"ericwbailey.website",
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "a11y-syntax-highlighting, a11y-webring.club, accessible-html-content-patterns, ericwbailey.website, cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 172,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "no"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got no stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 173,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "better"
},
"intent": "Open my latest updated issue that has keyword \"better\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/8",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": "",
"url_note": "GOLD in PRED"
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 174,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "feature"
},
"intent": "Open my latest updated issue that has keyword \"feature\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-webring.club/-/issues/71",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 175,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "dependency"
},
"intent": "Open my latest updated issue that has keyword \"dependency\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 176,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "theme editor"
},
"intent": "Open my latest updated issue that has keyword \"theme editor\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 177,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "homepage content"
},
"intent": "Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
"program_html": [],
"reference_answer_raw_annotation": "closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 178,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "better"
},
"intent": "Open my latest created issue that has better in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/566",
"program_html": [],
"reference_answer_raw_annotation": "Closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 179,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "feature"
},
"intent": "Open my latest created issue that has feature in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1517",
"program_html": [],
"reference_answer_raw_annotation": "Closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 180,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "dependency"
},
"intent": "Open my latest created issue that has dependency in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 181,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "theme editor"
},
"intent": "Open my latest created issue that has theme editor in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 182,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "homepage content"
},
"intent": "Open my latest created issue that has homepage content in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
"program_html": [],
"reference_answer_raw_annotation": "closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"shopping_admin"
],
"task_id": 183,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "SKU",
"N": "10"
},
"intent": "Give me the SKU of the products that have 10 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no product that has 10 quantities left.",
"reference_answer_raw_annotation": "There is no product that has 10 quantities left."
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 184,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "name",
"N": "0"
},
"intent": "Give me the name of the products that have 0 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Sinbad Fitness Tank"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sinbad Fitness Tank"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 185,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "brand",
"N": "3"
},
"intent": "Give me the brand of the products that have 3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Eos",
"Minerva"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Eos, Minerva"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 186,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "product names and the sizes",
"N": "2-3"
},
"intent": "Give me the product names and the sizes of the products that have 2-3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Eos V-Neck Hoodie: S",
"Minera Luma Tech V-Tee: XS"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Eos V-Neck Hoodie: S Minera Luma Tech V-Tee: XS"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 187,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "SKU",
"N": "1-3"
},
"intent": "Give me the SKU of the products that have 1-3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"WH11-S-Blue",
"WS08-XS-Blue"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "WH11-S-Blue, WS08-XS-Blue"
},
"intent_template_id": 368
},
{
"sites": [
"shopping"
],
"task_id": 188,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Tell me the total cost of my latest cancelled order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"365.42"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "365.42"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 189,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "pending"
},
"intent": "Tell me the total cost of my latest pending order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"754.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "754.99"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 190,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "complete"
},
"intent": "Tell me the total cost of my latest complete order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"65.32"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "65.32"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 191,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "processing"
},
"intent": "Tell me the total cost of my latest processing order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no order of \"processing\" status",
"reference_answer_raw_annotation": "There is no order of \"processing\" status"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 192,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "non-cancelled"
},
"intent": "Tell me the total cost of my latest non-cancelled order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"754.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "754.99"
},
"intent_template_id": 214
},
{
"sites": [
"shopping_admin"
],
"task_id": 193,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "completed",
"N": "2"
},
"intent": "Get the total payment amount of the last 2 completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"182.4"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "182.4"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 194,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "completed",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"555.2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "555.2"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 195,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "pending",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 pending orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"885.4"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "885.4"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 196,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Compare the payment difference of the last {{N}} {{status_1}} orders and {{status_2}} orders",
"instantiation_dict": {
"status_1": "cancelled",
"status_2": "completed",
"N": "4"
},
"intent": "Compare the payment difference of the last 4 cancelled orders and completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"194.25"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "194.25"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 197,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "non-cancelled",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 non-cancelled orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"778.2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"annotation_note": "219.4+210+166.4+93.4+89",
"reference_answer_raw_annotation": "778.2"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 198,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "customer name",
"status": "most recent cancelled"
},
"intent": "Get the customer name of the most recent cancelled order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Lily Potter"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lily Potter"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 199,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "order ID",
"status": "newest pending"
},
"intent": "Get the order ID of the newest pending order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"299"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "299"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 200,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "billing name",
"status": "oldest complete"
},
"intent": "Get the billing name of the oldest complete order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "John Lee"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "John Lee"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 201,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "customer name",
"status": "earliest fraud suspect"
},
"intent": "Get the customer name of the earliest fraud suspect order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no order of \"fraud suspect\" status",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 202,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "date",
"status": "most recent canlled"
},
"intent": "Get the date of the most recent canlled order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"May 23 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "May 23, 2023"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 203,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "purchase date and order id",
"status": "most recent pending"
},
"intent": "Get the purchase date and order id of the most recent pending order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"order id: 000000299",
"purchase date: May 31, 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000299, May 31, 2023, 2:55:09 AM"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 204,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "product name and discounted price (low to high)",
"status": "most recent completed"
},
"intent": "Get the product name and discounted price (low to high) of the most recent completed order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Rapha Sports Short: $35",
"Thorpe Track Pant: $54.4",
"Mach Street Sweatshirt: $62"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rapha Sports Short: $35 Thorpe Track Pant: $54.4 Mach Street Sweatshirt: $62"
},
"intent_template_id": 366
},
{
"sites": [
"gitlab"
],
"task_id": 205,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"date": "3/5/2023"
},
"intent": "How many commits did kilian make on 3/5/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 320
},
{
"sites": [
"gitlab"
],
"task_id": 206,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}}?",
"instantiation_dict": {
"user": "Eric",
"date": "3/2"
},
"intent": "How many commits did Eric make on 3/2?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 320
},
{
"sites": [
"gitlab"
],
"task_id": 207,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}} in total?",
"instantiation_dict": {
"user": "Eric and Kilian",
"date": "1/3/2023"
},
"intent": "How many commits did Eric and Kilian make on 1/3/2023 in total?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 320
},
{
"sites": [
"shopping_admin"
],
"task_id": 208,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "+1 2058812302"
},
"intent": "Find the customer name and email with phone number +1 2058812302",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"John Smith",
"john.smith.xyz@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "John Smith, john.smith.xyz@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 209,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "2137418080"
},
"intent": "Find the customer name and email with phone number 2137418080",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jennifer White",
"jennifer.white@yahoo.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jennifer White, jennifer.white@yahoo.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 210,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "2065555555"
},
"intent": "Find the customer name and email with phone number 2065555555",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Adam Garcia",
"gamingpro456@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Adam Garcia, gamingpro456@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 211,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "8015551212"
},
"intent": "Find the customer name and email with phone number 8015551212",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sean Miller",
"sean.miller@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sean Miller, sean.miller@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 212,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "555-229-3326"
},
"intent": "Find the customer name and email with phone number 555-229-3326",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Veronica Costello",
"roni_cost@example.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Veronica Costello, roni_cost@example.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 213,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "What are the key aspects that the customers don't like about Antonia Racer Tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Not suitable for high-impact workouts"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Not suitable for high-impact workouts"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 214,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Zing Jump Rope"
},
"intent": "What are the key aspects that the customers don't like about Zing Jump Rope",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"It is hard to find the right size. Won't last long"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "It is hard to find the right size. Won't last long"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 215,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Circe ice fleece"
},
"intent": "What are the key aspects that the customers don't like about Circe ice fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Material quality, fit, insufficient warmth, color"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Material quality, fit, insufficient warmth, color"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 216,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Electra Bra Top"
},
"intent": "What are the key aspects that the customers don't like about Electra Bra Top",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Not true to size"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Not true to size"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 217,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Pursuit Tone Band"
},
"intent": "What are the key aspects that the customers don't like about Pursuit Tone Band",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Insufficient resistance for their workouts."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Insufficient resistance for their workouts."
},
"intent_template_id": 249
},
{
"sites": [
"map"
],
"task_id": 218,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "CMU, Pittsburgh",
"n": "5"
},
"intent": "Show me the walking distance from nearby hotels to CMU, Pittsburgh that take at most 5 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 219,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "Pittsburgh airport",
"n": "3"
},
"intent": "Show me the walking distance from nearby hotels to Pittsburgh airport that take at most 3 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 220,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "Gardner Steel Conference Center,",
"n": 5
},
"intent": "Show me the walking distance from nearby hotels to Gardner Steel Conference Center, that take at most 5 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Wyndham Pittsburgh University Cente: 375m",
"The Oaklander Hotel: 338m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Wyndham Pittsburgh University Cente: 375 m\nThe Oaklander Hotel: 338 m"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 221,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to the nearest {{location}} with different transportation methods?",
"instantiation_dict": {
"location": "USPS postal office"
},
"intent": "I am at CMU Pittsburgh, how long it takes to the nearest USPS postal office with different transportation methods?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Walk: 1 minute",
"Drive: less than 1 minute",
"Bike: less than 1 minute"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Walk: 1 minute to walk and\nDrive: less than 1 minute\nBike: less than 1 minute"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 222,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "cold stone ice cream"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest cold stone ice cream",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3min"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 223,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "Mcdonald's"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest Mcdonald's",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 224,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "wendys"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest wendys",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3min"
},
"intent_template_id": 35
},
{
"sites": [
"shopping"
],
"task_id": 225,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What do customers say about {{product_type}} from {{manufature}}",
"instantiation_dict": {
"product_type": "brush",
"manufature": "sephora"
},
"intent": "What do customers say about brush from sephora",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The sephora brushes don't have reviews",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 135
},
{
"sites": [
"shopping"
],
"task_id": 226,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "Amazon basic"
},
"intent": "What is the price range for products from Amazon basic?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5.49",
"375.19"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$5.49 - $375.19"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 227,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "EYZUTAK"
},
"intent": "What is the price range for products from EYZUTAK?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$9.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 228,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "sephora"
},
"intent": "What is the price range for products from sephora?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"18.18",
"94.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$18.18 - $94.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 229,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "ugreen"
},
"intent": "What is the price range for products from ugreen?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"6.99",
"38.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$6.99 - $38.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 230,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "Perricone MD"
},
"intent": "What is the price range for products from Perricone MD?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"35",
"149"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$35 - $149"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 231,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Get the order number of my most recent cancelled order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"170"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000170"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 232,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "pending"
},
"intent": "Get the order number of my most recent pending order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"189"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000189"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 233,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "complete"
},
"intent": "Get the order number of my most recent complete order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"180"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000180"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 234,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "on hold"
},
"intent": "Get the order number of my most recent on hold order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "there is no on hold order",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 235,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "under delivery"
},
"intent": "Get the order number of my most recent under delivery order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no under delivery order",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 213
},
{
"sites": [
"map"
],
"task_id": 236,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
"instantiation_dict": {
"location": "pharmacy",
"location2": "Carnegie Mellon",
"condition": "I can walk within 20mins"
},
"intent": "Where is the nearest pharmacy from Carnegie Mellon I can walk within 20mins",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Schiller's Pharmacy",
"811",
"South Aiken Avenue",
"Shadyside",
"Pittsburgh"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Schiller's Pharmacy, 811, South Aiken Avenue, Shadyside, Pittsburgh, Allegheny County, 15232, United States"
},
"intent_template_id": 39
},
{
"sites": [
"map"
],
"task_id": 237,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
"instantiation_dict": {
"location": "gas station",
"location2": "CMU",
"condition": ""
},
"intent": "Where is the nearest gas station from CMU ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sunoco",
"North Craig Street",
"North Oakland",
"Pittsburgh"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sunoco, North Craig Street, North Oakland, Pittsburgh, Allegheny County, 15213, United States"
},
"intent_template_id": 39
},
{
"sites": [
"shopping"
],
"task_id": 238,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "PS4 accessories"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from PS4 accessories category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/astro-gaming-a50-wireless-headset-base-station-gen-4-compatible-with-ps5-ps4-pc-mac-black-silver.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 239,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "nutrition bars and drinks"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from nutrition bars and drinks category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/kellogg-s-special-k-protein-meal-bars-chocolate-caramel-12-7oz-6-count.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 240,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "competitive swimwear"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from competitive swimwear category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/women-cross-flower-beachwear-tankini-bandeau-bandage-bikini-set-push-up-swimwear-bathing-suit-two-pieces-swimsuits.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 241,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "skin care tool"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from skin care tool category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/professional-medi-spa-scar-stretch-mark-reduction-system.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 242,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "Household Supplies"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from Household Supplies category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/lynx-battery-12v-200ah-lithium-iron-phosphate-lifepo4-prismatic-deep-cell-battery-set-of-4-3-2v-cells-with-3-bus-bars-and-8-lug-nuts-for-rv-solar-marine-off-grid-applications.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping_admin"
],
"task_id": 243,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "Circe fleece"
},
"intent": "Show me the email address of the customer who is the most unhappy with Circe fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "hannah.lim@gmail.com"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hannah.lim@gmail.com"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 244,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "Olivia zip jacket"
},
"intent": "Show me the email address of the customer who is the most unhappy with Olivia zip jacket",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "emma.lopez@gmail.com"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "emma.lopez@gmail.com"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 245,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "name",
"product": "Antonia racer tank"
},
"intent": "Show me the name of the customer who is the most unhappy with Antonia racer tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Shaunte"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shaunte"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 246,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "name",
"product": "Chloe tank"
},
"intent": "Show me the name of the customer who is the most unhappy with Chloe tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Teofila"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Teofila"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 247,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "the style of Zoe products"
},
"intent": "Show me the email address of the customer who is the most unhappy with the style of Zoe products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "Valorie doesn't have a email in the system",
"program_html": [],
"string_note": "There is no negative review for Zoe products",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 244
},
{
"sites": [
"map"
],
"task_id": 248,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Carnegie Mellon Caf\u00e9"
},
"intent": "Tell me the coordinates of Carnegie Mellon Caf\u00e9 in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.442",
"-79.939"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4424191, -79.9397388"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 249,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Western Pennsylvania Hospital Heliport"
},
"intent": "Tell me the coordinates of Western Pennsylvania Hospital Heliport in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.460",
"-79.946"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.46076, -79.94666"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 250,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Apple Store near Pitt"
},
"intent": "Tell me the coordinates of Apple Store near Pitt in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.451",
"-79.933"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4511693, -79.9334241"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 251,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "bus stop on the Carnegie art museum side of the street near CMU"
},
"intent": "Tell me the coordinates of bus stop on the Carnegie art museum side of the street near CMU in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.444",
"-79.948"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4443, -79.94889"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 252,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Tokyo Japanese Food Store in Pittsburgh"
},
"intent": "Tell me the coordinates of Tokyo Japanese Food Store in Pittsburgh in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.457",
"-79.929"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.45761, -79.92934"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 253,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Carnegie Mellon Caf\u00e9",
"information": "phone number"
},
"intent": "What is the phone number of Carnegie Mellon Caf\u00e9",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no such information in the map",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 254,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Western Pennsylvania Hospital",
"information": "phone number"
},
"intent": "What is the phone number of Western Pennsylvania Hospital",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": ["4125785000"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4125785000"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 255,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Who is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "PIT airport",
"information": "operator"
},
"intent": "Who is the operator of PIT airport",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Allegheny County Airport Authority"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Allegheny County Airport Authority"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 256,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Carnegie art museum in pittsburgh",
"information": "website"
},
"intent": "What is the website of Carnegie art museum in pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "http://web.cmoa.org/"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "http://web.cmoa.org/"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 257,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Tokyo Japanese Food Store in Pittsburgh",
"information": "hours of operation"
},
"intent": "What is the hours of operation of Tokyo Japanese Food Store in Pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"We-Su 10:00-17:00"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "We-Su 10:00-17:00"
},
"intent_template_id": 501
},
{
"sites": [
"gitlab"
],
"task_id": 258,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "See all public projects",
"instantiation_dict": {},
"intent": "See all public projects",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/explore",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 325
},
{
"sites": [
"gitlab"
],
"task_id": 259,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Get me my RSS feed token",
"instantiation_dict": {},
"intent": "Get me my RSS feed token",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "TMN_bBn9Z48qVbUFZV45"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "TMN_bBn9Z48qVbUFZV45"
},
"intent_template_id": 312
},
{
"sites": [
"shopping"
],
"task_id": 260,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Video Game"
},
"intent": "I want to browse the products in the Video Game category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/video-games.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 261,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Headphones"
},
"intent": "I want to browse the products in the Headphones category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/electronics/headphones.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/electronics.html?cat=60"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 262,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Men shoes"
},
"intent": "I want to browse the products in the Men shoes category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=145"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 263,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Woman clothing"
},
"intent": "I want to browse the products in the Woman clothing category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/clothing.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=143"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 264,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Cabinets, Racks & Shelves"
},
"intent": "I want to browse the products in the Cabinets, Racks & Shelves category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/office-products/office-furniture-lighting/cabinets-racks-shelves.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/office-products.html?cat=187"}]
},
"intent_template_id": 211
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 265,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How far is it to drive there?",
"instantiation_dict": {
"city": "Boston"
},
"intent": "What's the closest national park to Boston? How far is it to drive there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park",
"457km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n457km"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 266,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}?",
"instantiation_dict": {
"city": "the largest city in Maine"
},
"intent": "What's the closest national park to the largest city in Maine?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Acadia National Park"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 267,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How long it takes to drive there?",
"instantiation_dict": {
"city": "the hometown of Stephen King"
},
"intent": "What's the closest national park to the hometown of Stephen King? How long it takes to drive there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park"
],
"fuzzy_match": [
"1h 23min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n1h 23min"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 268,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How long does it take to bike there?",
"instantiation_dict": {
"city": "Vinalhaven, ME"
},
"intent": "What's the closest national park to Vinalhaven, ME? How long does it take to bike there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park"
],
"fuzzy_match": [
"10h 33min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n10h 33min"
},
"intent_template_id": 85
},
{
"sites": [
"shopping"
],
"task_id": 269,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "25",
"product_category": "women shoes"
},
"intent": "Show me products under $25 in \"women shoes\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/shoes.html?price=0-25",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 270,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "30",
"product_category": "men shoes"
},
"intent": "Show me products under $30 in \"men shoes\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html?price=0-30",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 271,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "46.99",
"product_category": "makeup remover"
},
"intent": "Show me products under $46.99 in \"makeup remover\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/beauty-personal-care/makeup/makeup-remover.html?price=0-46.99",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 272,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "78",
"product_category": "children dental care"
},
"intent": "Show me products under $78 in \"children dental care\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/beauty-personal-care/oral-care/children-s-dental-care.html?price=0-78",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 273,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "199",
"product_category": "furtiture with accent"
},
"intent": "Show me products under $199 in \"furtiture with accent\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/furniture/accent-furniture.html?price=0-199",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 274,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "usb wifi"
},
"intent": "Search for \"usb wifi\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=usb+wifi",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 275,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "xbox"
},
"intent": "Search for \"xbox\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=xbox",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 276,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "switch accessories"
},
"intent": "Search for \"switch accessories\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=switch+accessories",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 277,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "batteries for iphone 13"
},
"intent": "Search for \"batteries for iphone 13\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=iphone+13",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 278,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "green tea bag for weight loss"
},
"intent": "Search for \"green tea bag for weight loss\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=green+tea+bag+for+weight+loss",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 279,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
"instantiation_dict": {},
"intent": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed)",
"Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones",
"Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus",
"Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black",
"Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items)",
"Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W)",
"Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B)",
"Sony MDRAS600BT Active Sports Bluetooth Headset (Black)",
"Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items)",
"Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)",
"Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black",
"SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed",
"18.99",
"406"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are avaiable: SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed) Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items) Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W) Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B) Sony MDRAS600BT Active Sports Bluetooth Headset (Black) Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items) Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R) Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed The price ranges from $18.99 to $406 "
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 280,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
"instantiation_dict": {},
"intent": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More",
"Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)",
"Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)",
"5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply",
"Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)",
"Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)",
"USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More",
"iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy",
"USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More",
"Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)",
"Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More",
"USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)",
"8.99",
"59.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included) Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278) 5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter) Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter) USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111) Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included) Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built-in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter) USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included) The price ranges from $8.99 to $59.99"
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 281,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
"instantiation_dict": {},
"intent": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count",
"Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack",
"3.745",
"6.495"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack The price ranges from $3.745 to $6.495 "
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 282,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
"instantiation_dict": {},
"intent": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Nike Men's Air Max Camden Slide Sandal",
"Nike Men's Benassi JDI Fanny Pack Slides",
"Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)",
"Nike Offcourt Slide Mens Bq4639-002 Size 12",
"Nike Jordan Men's Break Slide Red AR6374-602",
"Nike Victori One Slide Mens Style : Dd9559-300",
"Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)",
"Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)",
"Nike womens Benassi Just Do It",
"27.6",
"90.65"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Nike Men's Air Max Camden Slide Sandal Nike Men's Benassi JDI Fanny Pack Slides Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10) Nike Offcourt Slide Mens Bq4639-002 Size 12 Nike Jordan Men's Break Slide Red AR6374-602 Nike Victori One Slide Mens Style : Dd9559-300 Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14) Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8) Nike womens Benassi Just Do It The price ranges from $27.6 to $90.65"
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 283,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Look up the most recent models of XBox controllers released between 2020-2021?",
"instantiation_dict": {},
"intent": "Look up the most recent models of XBox controllers released between 2020-2021?",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/microsoft-xbox-controller-carbon-black-for-series-x-series-s-xbox-one-windows-10-android-ios-bundled-with-dual-port-charging-dock-xbox-controller-skin-voucher-premgear-cloth.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 210
},
{
"sites": [
"shopping"
],
"task_id": 284,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "shoe storage",
"min_storage": "12 pairs"
},
"intent": "Show the least expensive shoe storage with a minimum storage capacity of 12 pairs.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/onlyeasy-over-the-door-shoe-storage-organizer-hanging-shoe-rack-holder-with-24-large-fabric-pockets-22-1-x-61-4-herringbone-grey-mxrodsb1p.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"shopping"
],
"task_id": 285,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "switch card holder",
"min_storage": "15 cards"
},
"intent": "Show the least expensive switch card holder with a minimum storage capacity of 15 cards.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"shopping"
],
"task_id": 286,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "ssd hard drive",
"min_storage": "1TB"
},
"intent": "Show the least expensive ssd hard drive with a minimum storage capacity of 1TB.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/external-hard-drive-2tb-ultra-thin-external-hard-drive-2000gb-ultra-high-speed-portable-3-1-type-c-storage-drive-compatible-with-pc-laptop-and-mac-2tb-a1.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"map"
],
"task_id": 287,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How much time does it take from Pittsburgh to Philadelphia by car?",
"instantiation_dict": {},
"intent": "How much time does it take from Pittsburgh to Philadelphia by car?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"5h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5h 47min"
},
"intent_template_id": 47
},
{
"sites": [
"shopping_admin"
],
"task_id": 288,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "name"
},
"intent": "Tell me the name of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Samantha Jones"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Samantha Jones"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 289,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "email address, name, phone number"
},
"intent": "Tell me the email address, name, phone number of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"email: coolcat321@hotmail.com",
"name: Samantha Jones",
"phone number: 3055551212"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "email: coolcat321@hotmail.com name: Samantha Jones phone number: 3055551212"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 290,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "product SKUs in the most recent cancelled orders"
},
"intent": "Tell me the product SKUs in the most recent cancelled orders of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"WSH09-29-White",
"WSH09-28-Green",
"MSH11-34-Blue",
"WP09-29-Purple"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "WSH09-29-White,WSH09-28-Green,MSH11-34-Blue,WP09-29-Purple"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 291,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "total spend on products in the most recent cancelled orders"
},
"intent": "Tell me the total spend on products in the most recent cancelled orders of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"148"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$148"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 292,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "total number of cancellations"
},
"intent": "Tell me the total number of cancellations of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9"
},
"intent_template_id": 234
},
{
"sites": [
"gitlab"
],
"task_id": 293,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "Super_Awesome_Robot"
},
"intent": "Show me the command to clone Super_Awesome_Robot with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 294,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "ChatGPT"
},
"intent": "Show me the command to clone ChatGPT with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 295,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "metaseq"
},
"intent": "Show me the command to clone metaseq with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 296,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "the best GAN python implementation"
},
"intent": "Show me the command to clone the best GAN python implementation with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 297,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "the most stared Covid location tracker"
},
"intent": "Show me the command to clone the most stared Covid location tracker with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"
},
"intent_template_id": 329
},
{
"sites": [
"shopping"
],
"task_id": 298,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "completed"
},
"intent": "Show the most recent completed order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/180/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 299,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Show the most recent cancelled order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/170/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 300,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "pending"
},
"intent": "Show the most recent pending order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/189/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 301,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "processing"
},
"intent": "Show the most recent processing order page",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {"fuzzy_match": "N/A"},
"reference_url": "",
"program_html": [],
"string_note": "there is no order in processing"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 302,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "out of delivery"
},
"intent": "Show the most recent out of delivery order page",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {"fuzzy_match": "N/A"},
"reference_url": "",
"program_html": [],
"string_note": "there is no order in processing"
},
"intent_template_id": 180
},
{
"sites": [
"gitlab"
],
"task_id": 303,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Kilian",
"period": "durning 2023"
},
"intent": "How many commits did Kilian make durning 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 304,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Eric",
"period": "between Feb 2023 and May 2023"
},
"intent": "How many commits did Eric make between Feb 2023 and May 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"14"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "14"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 305,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Philip",
"period": "in 2023/1"
},
"intent": "How many commits did Philip make in 2023/1?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 306,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Anthony",
"period": "between 08/2022-09/2022"
},
"intent": "How many commits did Anthony make between 08/2022-09/2022?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 307,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Nic",
"period": "in April 2021"
},
"intent": "How many commits did Nic make in April 2021?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 308,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "primer/design"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the primer/design project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Shawn Allen"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shawn Allen"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 309,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "thoughtbot/administrate"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the thoughtbot/administrate project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Grayson Wright"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Grayson Wright"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 310,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "AndroidSlidingUpPanel"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the AndroidSlidingUpPanel project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["tokudu"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "tokudu"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 311,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "Pytorch GAN"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the Pytorch GAN project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Erik Linder-Nor\u00e9n"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Erik Linder-Nor\u00e9n"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 312,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "csvkit"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the csvkit project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Christopher Groskopf"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Christopher Groskopf"
},
"intent_template_id": 323
},
{
"sites": [
"shopping"
],
"task_id": 313,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Which number to call for the customer service?",
"instantiation_dict": {},
"intent": "Which number to call for the customer service?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no phone number in the website",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 134
},
{
"sites": [
"gitlab"
],
"task_id": 314,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "prime/design",
"attribute": "name"
},
"intent": "List the name of the top 3 contributors to prime/design repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Shawn Allen",
"Inayaili Le\u00f3n",
"Aurora Pleguezuelo"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shawn Allen, Inayaili Le\u00f3n, Aurora Pleguezuelo"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 315,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "Pytorch GAN",
"attribute": "email address"
},
"intent": "List the email address of the top 3 contributors to Pytorch GAN repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"eriklindernoren@live.se",
"eriklindernoren@gmail.com",
"pinnacle.chen@qq.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "eriklindernoren@live.se, eriklindernoren@gmail.com, pinnacle.chen@qq.com"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 316,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "facebook's guide on building react apps",
"attribute": "name"
},
"intent": "List the name of the top 3 contributors to facebook's guide on building react apps repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Ian Sutherland",
"Joe Hadda",
"Dan Abramov"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Ian Sutherland, Joe Hadda, Dan Abramov"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 317,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "metaseq",
"attribute": "name and number of commits"
},
"intent": "List the name and number of commits of the top 3 contributors to metaseq repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Susan Zhang: 70",
"Stephen Roller: 51",
"Peter Albert: 12"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Susan Zhang: 70, Stephen Roller: 51, Peter Albert: 12"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 318,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "2019-nCov",
"attribute": "last names"
},
"intent": "List the last names of the top 3 contributors to 2019-nCov repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lo",
"Chen",
"Chu"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lo, Chen, Chu"
},
"intent_template_id": 324
},
{
"sites": [
"shopping"
],
"task_id": 319,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "April 2022"
},
"intent": "How much refund I should expect from my order canlled in April 2022, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 320,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "Feb 2023"
},
"intent": "How much refund I should expect from my order canlled in Feb 2023, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"406.53"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "406.53"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 321,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "2022"
},
"intent": "How much refund I should expect from my order canlled in 2022, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"3053.97"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3053.97"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 322,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}} if I cannot get the shipping fee refunded?",
"instantiation_dict": {
"time": "May 2023"
},
"intent": "How much refund I should expect from my order canlled in May 2023 if I cannot get the shipping fee refunded?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"350.42"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "350.42"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 323,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
"instantiation_dict": {
"time": "2022/03"
},
"intent": "How much refund I should expect from my order canlled in 2022/03? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"264.49"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "264.49"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 324,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "chairs",
"sorting_order": "ascending price"
},
"intent": "Show me the \"chairs\" listings by ascending price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=chairs&product_list_dir=asc",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 325,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "mouth night guard",
"sorting_order": "descending price"
},
"intent": "Show me the \"mouth night guard\" listings by descending price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?q=mouth%20night%20guard%20&product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 326,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "Canon photo printer",
"sorting_order": "search relevance, from most to least"
},
"intent": "Show me the \"Canon photo printer\" listings by search relevance, from most to least.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=Canon+photo+printer",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 327,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "iphone 12 phone case",
"sorting_order": "name alphabetically"
},
"intent": "Show me the \"iphone 12 phone case\" listings by name alphabetically.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?q=%20iphone%2012%20phone%20case&product_list_order=name",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 328,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "iphone 12 phone case",
"sorting_order": "price"
},
"intent": "Show me the \"iphone 12 phone case\" listings by price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=%20iphone%2012%20phone%20case",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 329,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "on 4/19/2023"
},
"intent": "How much I spend on 4/19/2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 330,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "in March 2023"
},
"intent": "How much I spend in March 2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"83.31"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "81.31"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 331,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "in July 2022"
},
"intent": "How much I spend in July 2022 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.16"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 332,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "each month from Jan to the end of March 2023"
},
"intent": "How much I spend each month from Jan to the end of March 2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Jan: 572.8",
"Feb: 762.18",
"Mar: 83.31"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jan: 572.8\nFeb: 762.18\nMar: 83.31"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 333,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much did I spend on shopping at One Stop Market {{time}}? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
"instantiation_dict": {
"time": "on November 2022"
},
"intent": "How much did I spend on shopping at One Stop Market on November 2022? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"359.546"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "359.546"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 334,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "muffin cornbread mix"
},
"intent": "Tell me when I last ordered my muffin cornbread mix?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"March 11th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "March 11th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 335,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "body butter"
},
"intent": "Tell me when I last ordered my body butter?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January 16th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January 16th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 336,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "conditioner"
},
"intent": "Tell me when I last ordered my conditioner?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January 16th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January 16th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 337,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "bread olive"
},
"intent": "Tell me when I last ordered my bread olive?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"December 12th 2022"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "December 12th 2022"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 338,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "toothpaste"
},
"intent": "Tell me when I last ordered my toothpaste?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"December 4th 2022"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "December 4th 2022"
},
"intent_template_id": 169
},
{
"sites": [
"gitlab"
],
"task_id": 339,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that report bugs"
},
"intent": "List all opened issues that report bugs",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?label_name%5B%5D=bug",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 340,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that report bugs"
},
"intent": "List all opened issues that report bugs",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/issues/?label_name%5B%5D=type%3A%20bug%20%F0%9F%90%9E",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 341,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "requesting new features"
},
"intent": "List all opened issues requesting new features",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=enhancement",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 342,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that ask about OPT model related questions"
},
"intent": "List all opened issues that ask about OPT model related questions",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?search=OPT&label_name%5B%5D=question",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 343,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that don't have any labels"
},
"intent": "List all opened issues that don't have any labels",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=None",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"shopping_admin"
],
"task_id": 344,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "by far"
},
"intent": "How many reviews our shop received by far?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 345,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "in Apr 2023"
},
"intent": "How many reviews our shop received in Apr 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 346,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "during 2022"
},
"intent": "How many reviews our shop received during 2022?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 347,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "from the beginning of the shop"
},
"intent": "How many reviews our shop received from the beginning of the shop?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 348,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "in May 2023"
},
"intent": "How many reviews our shop received in May 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 248
},
{
"sites": [
"gitlab"
],
"task_id": 349,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
"instantiation_dict": {
"repo": "gimmiethat.space"
},
"intent": "Who else have access to my repo gimmiethat.space, show me their usernames",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "yjlou"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "yjlou"
},
"intent_template_id": 298
},
{
"sites": [
"gitlab"
],
"task_id": 350,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
"instantiation_dict": {
"repo": "prism-theme"
},
"intent": "Who else have access to my repo prism-theme, show me their usernames",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"abisubramanya27"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Abishek S, abisubramanya27"
},
"intent_template_id": 298
},
{
"sites": [
"shopping"
],
"task_id": 351,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "PS4 accessories",
"order": "ascending"
},
"intent": "List products from PS4 accessories category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/video-games/playstation-4/accessories.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/video-games.html?cat=67&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 352,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "nutrition bars and drinks",
"order": "ascending"
},
"intent": "List products from nutrition bars and drinks category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/health-household/diet-sports-nutrition/nutrition-bars-drinks.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/health-household.html?cat=192&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 353,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "competitive swimwear",
"order": "ascending"
},
"intent": "List products from competitive swimwear category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/sport-specific-clothing/competitive-swimwear.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=149&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 354,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "living room furtniture",
"order": "descending"
},
"intent": "List products from living room furtniture category by descending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/furniture/living-room-furniture.html?product_list_order=price&product_list_dir=desc",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=154&product_list_order=price&product_list_dir=desc"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 355,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "kids' bedding",
"order": "descending"
},
"intent": "List products from kids' bedding category by descending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/bedding/kids-bedding.html?product_list_dir=desc",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=155&product_list_dir=desc"}]
},
"intent_template_id": 137
},
{
"sites": [
"map"
],
"task_id": 356,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
"instantiation_dict": {},
"intent": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Gates and Hillman Centers",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Independence Hall",
"Philadelphia"
]
}
}
]
},
"intent_template_id": 49
},
{
"sites": [
"gitlab"
],
"task_id": 357,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Checkout merge requests requiring my review",
"instantiation_dict": {},
"intent": "Checkout merge requests requiring my review",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/merge_requests?reviewer_username=byteblaze",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 291
},
{
"sites": [
"shopping"
],
"task_id": 358,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "shipping method",
"order_number": 187
},
"intent": "Show me the shipping method for order number 187.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Flat Rate - Fixed"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Flat Rate - Fixed"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 359,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "order date",
"order_number": "148"
},
"intent": "Show me the order date for order number 148.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"1/29/2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1/29/2023"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 360,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "product names",
"order_number": "148"
},
"intent": "Show me the product names for order number 148.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress)",
"Russound 5B45W 4\" Indoor Outdoor Speakers White"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress), Russound 5B45W 4\" Indoor Outdoor Speakers White"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 361,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "order statuses",
"order_number": "170 and 189"
},
"intent": "Show me the order statuses for order number 170 and 189.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"170: cancelled",
"189: pending"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "170: cancelled, 189: pending"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 362,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "billing address",
"order_number": "00178"
},
"intent": "Show me the billing address for order number 00178.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"101 S San Mateo Dr",
"San Mateo",
"California",
"94010",
"United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Emma Lopez, 101 S San Mateo Dr, San Mateo, California, 94010, United States"
},
"intent_template_id": 206
},
{
"sites": [
"map"
],
"task_id": 363,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "Carnegie Music Hall"
},
"intent": "Measure distance between Carnegie Mellon University and Carnegie Music Hall by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["778m |OR| 778 m"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "748m"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 364,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between Carnegie Mellon University and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.7km |OR| 1.7 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.7km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 365,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Music Hall",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between Carnegie Music Hall and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["2.2km |OR| 2.2 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2.2km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 366,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "CVS (closet one)",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between CVS (closet one) and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.2km |OR| 1.2 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.2km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 367,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "CVS (closet one)"
},
"intent": "Measure distance between Carnegie Mellon University and CVS (closet one) by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.4km |OR| 1.4 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.4km"
},
"intent_template_id": 58
},
{
"sites": [
"shopping"
],
"task_id": 368,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "find discounted items.",
"instantiation_dict": {},
"intent": "find discounted items.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no function to show only discount items",
"reference_answer_raw_annotation": "There is no function to show only discount items."
},
"intent_template_id": 188
},
{
"sites": [
"map"
],
"task_id": 369,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Carnegie Music Hall"
},
"intent": "Pull up the description page of Carnegie Music Hall on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Music Hall"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 370,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Carnegie Mellon University"
},
"intent": "Pull up the description page of Carnegie Mellon University on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Mellon University"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 371,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Piada restaurant near Pitt"
},
"intent": "Pull up the description page of Piada restaurant near Pitt on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Piada Italian Street Food",
"Forbes Avenue"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 372,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "the Costco in Pittsburhg near a river"
},
"intent": "Pull up the description page of the Costco in Pittsburhg near a river on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Costco",
"Waterfront Drive West"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 373,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Whole Foods near Carnegie Mellon"
},
"intent": "Pull up the description page of Whole Foods near Carnegie Mellon on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Whole Foods",
"East Liberty"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"shopping_admin"
],
"task_id": 374,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Preview the {{name}} theme for my shop",
"instantiation_dict": {
"name": "Magento Blank"
},
"intent": "Preview the Magento Blank theme for my shop",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/1",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 266
},
{
"sites": [
"shopping_admin"
],
"task_id": 375,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Preview the {{name}} theme for my shop",
"instantiation_dict": {
"name": "Magento Luma"
},
"intent": "Preview the Magento Luma theme for my shop",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/3/key/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 266
},
{
"sites": [
"shopping"
],
"task_id": 376,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Summarize customer reviews for {{product}}.",
"instantiation_dict": {
"product": "Amazon Echo Dot 3rd generation"
},
"intent": "Summarize customer reviews for Amazon Echo Dot 3rd generation.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no review for this product",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 182
},
{
"sites": [
"map"
],
"task_id": 377,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU ArtPark Lab",
"space": "resturants"
},
"intent": "Find the resturants around CMU ArtPark Lab",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=restaurants%20near%20CMU%20ArtPark%20Lab",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 378,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU main campus",
"space": "parking"
},
"intent": "Find the parking around CMU main campus",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=parking%20near%20carnegie%20mellon%20university",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 379,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU main campus",
"space": "hotel"
},
"intent": "Find the hotel around CMU main campus",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=hotels%20near%20carnegie%20mellon%20university",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 380,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "Carnegie Music Hall",
"space": "bar"
},
"intent": "Find the bar around Carnegie Music Hall",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=bars%20near%20Carnegie%20Music%20Hall",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 381,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "Carnegie Music Hall",
"space": "hotel"
},
"intent": "Find the hotel around Carnegie Music Hall",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=hotels%20near%20Carnegie%20Music%20Hall",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 382,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
"instantiation_dict": {},
"intent": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no USCIS nearby",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 781
},
{
"sites": [
"map"
],
"task_id": 383,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
"instantiation_dict": {},
"intent": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport",
"Giant Eagle",
"ALDI"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Giant Eagle, ALDI"
},
"intent_template_id": 782
},
{
"sites": [
"shopping"
],
"task_id": 384,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the customer names who complain about the quality of EYZUTAK phone cases",
"instantiation_dict": {},
"intent": "List the customer names who complain about the quality of EYZUTAK phone cases",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Lee",
"Evelyn Kurver",
"Amanda",
"N Randall"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Lee, Evelyn Kurver, Amanda, N Randall"
},
"intent_template_id": 666
},
{
"sites": [
"shopping"
],
"task_id": 385,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the customer names who thinks EYZUTAK phone cases are of good looking",
"instantiation_dict": {},
"intent": "List the customer names who thinks EYZUTAK phone cases are of good looking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Lee",
"MH",
"Misba009",
"Amanda",
"N Randall",
"Amazon Customer",
"Cally",
"Bethany Robertson"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Lee, MH, Misba009, Amanda, N Randall, Amazon Customer, Cally, Bethany Robertson"
},
"intent_template_id": 666
},
{
"sites": [
"shopping"
],
"task_id": 386,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the rating of {{product}}",
"instantiation_dict": {
"product": "Ugreen lightning to 3.5mm cable"
},
"intent": "What is the rating of Ugreen lightning to 3.5mm cable. Please round to the nearest whole number",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"65 |OR| 3"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Around 3.1 stars"
},
"intent_template_id": 1355
},
{
"sites": [
"shopping"
],
"task_id": 387,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
"instantiation_dict": {
"stars": "4 or 5 stars"
},
"intent": "Who gave 4 or 5 stars for phone cases from EYZUTAK",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"MH",
"Misba009",
"Amanda",
"Amazon Customer",
"Cally",
"Bethany Robertson"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "MH, Misba009, Amanda, Amazon Customer, Cally, Bethany Robertson, Bethany Robertson"
},
"intent_template_id": 1356
},
{
"sites": [
"shopping"
],
"task_id": 388,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
"instantiation_dict": {
"stars": "1 or 2 stars"
},
"intent": "Who gave 1 or 2 stars for phone cases from EYZUTAK",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Evelyn Kurver",
"N Randall"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Evelyn Kurver, N Randall"
},
"intent_template_id": 1356
},
{
"sites": [
"gitlab"
],
"task_id": 389,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "octovisuals page",
"repo": "primer/design",
"content": "Thanks, working on reviews"
},
"intent": "Post \"Thanks, working on reviews\" for the merge request related to octovisuals page in primer/design project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/primer/design/-/merge_requests/450",
"locator": "",
"required_contents": {
"must_include": [
"Thanks, working on reviews"
]
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 390,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "semantic HTML post",
"repo": "a11yproject/a11yproject.com",
"content": "lgtm"
},
"intent": "Post \"lgtm\" for the merge request related to semantic HTML post in a11yproject/a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1531",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "lgtm"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 391,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "focus edge cases",
"repo": "a11yproject/a11yproject.com",
"content": "close because non reproducible"
},
"intent": "Post \"close because non reproducible\" for the merge request related to focus edge cases in a11yproject/a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1265",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "close because non reproducible"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 392,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "color ulitity",
"repo": "a11yproject.com",
"content": "Good idea"
},
"intent": "Post \"Good idea\" for the merge request related to color ulitity in a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1071",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "Good idea"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 393,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "fixing the broken links",
"repo": "byteblaze/empathy-prompts",
"content": "lgtm"
},
"intent": "Post \"lgtm\" for the merge request related to fixing the broken links in byteblaze/empathy-prompts project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/empathy-prompts/-/merge_requests/19",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "lgtm"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 394,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "2019-nCov"
},
"intent": "Fork 2019-nCov.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/2019-nCov",
"locator": "",
"required_contents": {
"must_include": [
"2019-nCov"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 395,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "the Pytorch GAN repo with most stars"
},
"intent": "Fork the Pytorch GAN repo with most stars.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/PyTorch-GAN",
"locator": "",
"required_contents": {
"must_include": [
"Pytorch-GAN"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 396,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "ChatGPT"
},
"intent": "Fork ChatGPT.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/ChatGPT",
"locator": "",
"required_contents": {
"must_include": [
"ChatGPT"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 397,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "MetaSeq"
},
"intent": "Fork MetaSeq.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/metaseq",
"locator": "",
"required_contents": {
"must_include": [
"metaseq"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 398,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "all source repos from Akilesh Kannan"
},
"intent": "Fork all source repos from Akilesh Kannan",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/SimCache",
"locator": "",
"required_contents": {
"must_include": [
"SimCache"
]
}
},
{
"url": "__GITLAB__/byteblaze/dots",
"locator": "",
"required_contents": {
"must_include": [
"dots"
]
}
},
{
"url": "__GITLAB__/byteblaze/CacheEval",
"locator": "",
"required_contents": {
"must_include": [
"CacheEval"
]
}
},
{
"url": "__GITLAB__/byteblaze/nvidia-patch",
"locator": "",
"required_contents": {
"must_include": [
"404"
]
}
},
{
"url": "__GITLAB__/byteblaze/viewgrades-scraper",
"locator": "",
"required_contents": {
"must_include": [
"404"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"reddit"
],
"task_id": 399,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "I am a robot"
},
"intent": "Change my reddit bio to \"I am a robot\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "I am a robot"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 400,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Pro Python Developer with 20 years of Experience"
},
"intent": "Change my reddit bio to \"Pro Python Developer with 20 years of Experience\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Pro Python Developer with 20 years of Experience"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 401,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Seeking SDE positions"
},
"intent": "Change my reddit bio to \"Seeking SDE positions\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Seeking SDE positions"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 402,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Freelance Web Developer"
},
"intent": "Change my reddit bio to \"Freelance Web Developer\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Freelance Web Developer"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 403,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Awesome Prompt Artist"
},
"intent": "Change my reddit bio to \"Awesome Prompt Artist\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Awesome Prompt Artist"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 404,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "books"
},
"intent": "Upvote the newest post in books subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/books/124260/adults-reading-to-each-other-out-loud",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 405,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "DIY"
},
"intent": "Upvote the newest post in DIY subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/DIY/119019/how-can-i-bring-an-hdmi-cable-from-my-pc-downstairs-to-my-tv",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 406,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "future technology"
},
"intent": "Upvote the newest post in future technology subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/Futurology/119517/openai-ceo-it-s-not-funny-that-i-m-afraid-of-the-ai-we-re",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 407,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "deeplearning"
},
"intent": "Upvote the newest post in deeplearning subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/deeplearning/124993/meta-s-llama-weights-leaked-on-torrent-and-the-best-thing",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 408,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "explain like im 5"
},
"intent": "Upvote the newest post in explain like im 5 subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/explainlikeimfive/39244/eli5-how-does-pinching-a-ribbon-and-sliding-your-finger",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 409,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
"geolocation": null,
"intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the manager of the website",
"content_description": "thanks! I am a big fan of your website."
},
"intent": "Reply to the manager of the website in this post with \"thanks! I am a big fan of your website.\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/books/59421/-/comment/1235250",
"locator": "",
"required_contents": {
"must_include": [
"thanks! I am a big fan of your website."
]
}
}
]
},
"intent_template_id": 23
},
{
"sites": [
"reddit"
],
"task_id": 410,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/singularity/69404/this-is-how-chatgpt-sees-itself",
"geolocation": null,
"intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the first reply",
"content_description": "don't panic"
},
"intent": "Reply to the first reply in this post with \"don't panic\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/singularity/69404/-/comment/1042264",
"locator": "",
"required_contents": {
"must_include": [
"don't panic"
]
}
}
]
},
"intent_template_id": 23
},
{
"sites": [
"gitlab"
],
"task_id": 411,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to MIT license.",
"instantiation_dict": {
"repo": "byteblaze/cloud-to-butt"
},
"intent": "Make the LICENSE of byteblaze/cloud-to-butt to MIT license.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/cloud-to-butt/-/blob/master/LICENSE.txt",
"locator": "",
"required_contents": {
"must_include": [
"MIT license"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 412,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to Apache License",
"instantiation_dict": {
"repo": "byteblaze/accessible-html-content-patterns"
},
"intent": "Make the LICENSE of byteblaze/accessible-html-content-patterns to Apache License",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/accessible-html-content-patterns/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"Apache License",
"http://www.apache.org/licenses/LICENSE-2.0"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 413,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to one that mandates all copies and derivative works to be under the same license",
"instantiation_dict": {
"repo": "byteblaze/a11y-syntax-highlighting"
},
"intent": "Make the LICENSE of byteblaze/a11y-syntax-highlighting to one that mandates all copies and derivative works to be under the same license",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"GENERAL PUBLIC LICENSE"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 414,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to MIT license.",
"instantiation_dict": {
"repo": "byteblaze/dotfiles"
},
"intent": "Make the LICENSE of byteblaze/dotfiles to MIT license.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"MIT license",
"The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 415,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "verification functions"
},
"intent": "Go to the merge request on verification functions I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/a11y-webring.club/-/merge_requests/40",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["@davepgreene"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 416,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "wcag"
},
"intent": "Go to the merge request on wcag I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1270",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["Thank you"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 417,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "404 link"
},
"intent": "Go to the merge request on 404 link I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1485",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["@Roshanjossey"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 418,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Busy"
},
"intent": "Set my gitlab status as Busy.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-title').textContent",
"required_contents": {
"must_include": ["Busy"]
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 419,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Enjoying life"
},
"intent": "Set my gitlab status as Enjoying life.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Enjoying life"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 420,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Playing Badminton"
},
"intent": "Set my gitlab status as Playing Badminton.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Playing Badminton"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 421,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Resting due to leg injury"
},
"intent": "Set my gitlab status as Resting due to leg injury.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Resting due to leg injury"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 422,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Out of Office"
},
"intent": "Set my gitlab status as Out of Office.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Out of Office"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"shopping_admin"
],
"task_id": 423,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Mark all {{brand}} shirts on sale",
"instantiation_dict": {
"brand": "Hollister"
},
"intent": "Mark all Hollister shirts on sale",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/126/",
"locator": "document.querySelector('input[name=\"product[sale]\"]').value",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 237
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 424,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the place where Mr. Rogers was filmed"
},
"intent": "Find the page of the place where Mr. Rogers was filmed on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Pittsburgh"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 425,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the longest bridge in the Western hemisphere"
},
"intent": "Find the page of the longest bridge in the Western hemisphere on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Mackinac Bridge"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 426,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the place in Pennsylvania where a plane crashed during the September 11th attacks"
},
"intent": "Find the page of the place in Pennsylvania where a plane crashed during the September 11th attacks on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Somerset County"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 427,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the university that has most Turning Award winners"
},
"intent": "Find the page of the university that has most Turning Award winners on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Massachusetts Institute of Technology"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 428,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the undergrad college of the person who developed the Nash equilibrium"
},
"intent": "Find the page of the undergrad college of the person who developed the Nash equilibrium on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Mellon University"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 429,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the colleges where The Chair was filmed in Pittsburgh"
},
"intent": "Find the page of the colleges where The Chair was filmed in Pittsburgh on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Chatham University"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 430,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh"
},
"intent": "Find the page of the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Washington & Jefferson College"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"shopping"
],
"task_id": 431,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/tall-pink-taper-candles-4-piece-orange-colored-tapered-candles-gradient-candles-10-6-inches-tall-tie-dye-candle-set-large-dripless-long-burning-candlesticks-two-color-taper-candles-candlesticks.html |AND| __SHOPPING__/spaas-white-taper-candles-4-pack-10-inch-tall-candles-scent-free-premium-wax-candle-sticks-8-hour-long-burning-white-candlesticks-for-home-decoration-wedding-holiday-and-parties.html |AND| __SHOPPING__/white-starfish-wall-candle-sconces-set-of-2-beach-decor-ocean-themed-wall-mount-candleholders-nautical-style-beach-bathroom-decor-coastal-farmhouse-seashell-candle-holders.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"SPAAS White Taper Candles - 4 Pack |OR| 10 Inch Tall Candles, Scent-Free Premium Wax Candle Sticks |OR| 8 Hour Long Burning White Candlesticks for Home Decoration, Wedding, Holiday and Parties"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 432,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/ciclon-energy-drink-regular-24-cans-8-3oz.html |AND| __SHOPPING__/v8-energy-healthy-energy-drink-steady-energy-from-black-and-green-tea-pomegranate-blueberry-8-ounce-can-pack-of-24.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"V8 +Energy, Healthy Energy Drink, Steady Energy from Black and Green Tea, Pomegranate Blueberry, 8 Ounce Can ,Pack of 24"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 433,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/tazrigo-5pcs-white-dental-resin-brush-pens-dental-shaping-silicone-tooth-tool.html |AND| __SHOPPING__/stylus-pens-for-touch-screens-2-pcs-universal-stylus-2-in-1-2022-updated-touch-screen-pens-for-all-touch-screens-cell-phones-tablets-laptops-with-6-replacement-tips-4-discstips-2-fiber-tips.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"Tazrigo 5pcs White Dental Resin Brush Pens Dental Shaping Silicone Tooth Tool"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 434,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/3-pairs-ruffle-socks-lace-ankle-socks-for-girls-frilly-socks-women-decorative.html |AND| __SHOPPING__/viviki-women-glitter-socks-ultrathin-transparent-tulle-lace-socks-no-show-ankle-crew-socks-3-pack.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"VIVIKI Women Glitter Socks Ultrathin Transparent Tulle Lace Socks - No Show Ankle Crew Socks 3 Pack"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 435,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/35-ft-hdmi-cable-gearit-pro-series-hdmi-cable-35-feet-high-speed-ethernet-4k-resolution-3d-video-and-arc-audio-return-channel-hdmi-cable-white.html |AND| __SHOPPING__/dp-to-hdmi-cable-6ft-2-pack-fosmon-gold-plated-displayport-to-hdmi-cable-1080p-full-hd-for-pcs-to-hdtv-monitor-projector-with-hdmi-port.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"DP to HDMI Cable 6FT (2 Pack), Fosmon Gold Plated Displayport to HDMI Cable 1080p Full HD for PCs to HDTV, Monitor, Projector with HDMI Port"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 436,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a mattress foundation",
"time": "around Feb or March 2023"
},
"intent": "I previously ordered some a mattress foundation around Feb or March 2023 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B07DFJ5XKH"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 437,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a table lamp",
"time": "in May 2023"
},
"intent": "I previously ordered some a table lamp in May 2023 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B072XS3F6W"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 438,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a TV stand",
"time": "sometime around sep 2022"
},
"intent": "I previously ordered some a TV stand sometime around sep 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B08PVHRRB7"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 439,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a cat t-shirt",
"time": "during 2022"
},
"intent": "I previously ordered some a cat t-shirt during 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B0844BWS76"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 440,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a make up removal kit",
"time": "during summer 2022"
},
"intent": "I previously ordered some a make up removal kit during summer 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B0738JQG6Q"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"gitlab"
],
"task_id": 441,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "GIVE ME SPACE"
},
"intent": "Update the project site's title to \"GIVE ME SPACE\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>GIVE ME SPACE</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 442,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Welcome to my site"
},
"intent": "Update the project site's title to \"Welcome to my site\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Welcome to my site</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 443,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Not an interesting site"
},
"intent": "Update the project site's title to \"Not an interesting site\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Not an interesting site</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 444,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Title Wanted"
},
"intent": "Update the project site's title to \"Title Wanted\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Title Wanted</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 445,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Hello"
},
"intent": "Update the project site's title to \"Hello\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Hello</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 446,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
"instantiation_dict": {
"repo": "a11yproject",
"issue": 404,
"account": "Roshanjossey"
},
"intent": "Assign the issue regarding 404 in a11yproject to Roshanjossey.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Roshanjossey",
"locator": "",
"required_contents": {
"must_include": [
"404s, bad host, timeouts, bad urls for URLs linked from website"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 447,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
"instantiation_dict": {
"repo": "a11y-webring.club",
"issue": "linking to an accessibility statement",
"account": "Rohan"
},
"intent": "Assign the issue regarding linking to an accessibility statement in a11y-webring.club to Rohan.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Seirdy",
"locator": "",
"required_contents": {
"must_include": [
"linking to an accessibility statement"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 448,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "https://egg.tart.com"
},
"intent": "set the homepage URL on my GitLab profile to https://egg.tart.com",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "egg.tart.com"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 449,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "https://helloworld.xyz"
},
"intent": "set the homepage URL on my GitLab profile to https://helloworld.xyz",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "helloworld.xyz"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 450,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "a11yproject.contributor.me"
},
"intent": "set the homepage URL on my GitLab profile to a11yproject.contributor.me",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "a11yproject.contributor.me"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 451,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "www.byteblaze.com"
},
"intent": "set the homepage URL on my GitLab profile to www.byteblaze.com",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "www.byteblaze.com"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 452,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "byteblaze.github.io"
},
"intent": "set the homepage URL on my GitLab profile to byteblaze.github.io",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "byteblaze.github.io"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"shopping_admin"
],
"task_id": 453,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Teton pullover hoodie"
},
"intent": "Disable Teton pullover hoodie from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/78/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 454,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Ryker Tee Crew Neck"
},
"intent": "Disable Ryker Tee Crew Neck from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/478/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 455,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "lHelios Endurance Tank"
},
"intent": "Disable lHelios Endurance Tank from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/676/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 456,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Cora Pant"
},
"intent": "Disable Cora Pant from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1840/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 457,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Karmen yoga pants"
},
"intent": "Disable Karmen yoga pants from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1819/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 458,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "$5",
"action": "Reduce"
},
"intent": "Reduce the price of this product by $5",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "27.00"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 459,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "10%",
"action": "Reduce"
},
"intent": "Reduce the price of this product by 10%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "62.10"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 460,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "15%",
"action": "Reduce"
},
"intent": "Reduce the price of this product by 15%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "38.25"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 461,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "$11.5",
"action": "Increase"
},
"intent": "Increase the price of this product by $11.5",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "29.50"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 462,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "10%",
"action": "Increase"
},
"intent": "Increase the price of this product by 10%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "19.80"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 463,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "15%",
"action": "Increase"
},
"intent": "Increase the price of this product by 15%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "36.80"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 464,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the product description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "Update the product description of Antonia Racer Tank to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../antonia-racer-tank.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"This is in regular rotation at the gym",
"Its colorful and looks kinda cute under my exercise tanks",
"it's very stylish for yoga or something else low impact"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping"
],
"task_id": 465,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
},
"intent": "Add Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 466,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
},
"intent": "Add 2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 467,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
},
"intent": "Add HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 468,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
},
"intent": "Add DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 469,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
},
"intent": "Add Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping_admin"
],
"task_id": 470,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "302"
},
"intent": "Cancel order 302",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 471,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "307"
},
"intent": "Cancel order 307",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/307/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 472,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "299"
},
"intent": "Cancel order 299",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/299/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 473,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "301"
},
"intent": "Cancel order 301",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/301/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 474,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "305"
},
"intent": "Cancel order 305",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/305/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"gitlab"
],
"task_id": 475,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "chatgpt_plugin"
},
"intent": "Set up a new, empty repository with the name chatgpt_plugin?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/chatgpt_plugin",
"locator": "",
"required_contents": {
"must_include": [
"chatgpt_plugin"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 476,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_llm_reading"
},
"intent": "Set up a new, empty repository with the name awesome_llm_reading?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_llm_reading",
"locator": "",
"required_contents": {
"must_include": [
"awesome_llm_reading"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 477,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_program_aided_reasoning"
},
"intent": "Set up a new, empty repository with the name awesome_program_aided_reasoning?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_program_aided_reasoning",
"locator": "",
"required_contents": {
"must_include": [
"awesome_program_aided_reasoning"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 478,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "webagent"
},
"intent": "Set up a new, empty repository with the name webagent?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/webagent",
"locator": "",
"required_contents": {
"must_include": [
"webagent"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 479,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_webagent"
},
"intent": "Set up a new, empty repository with the name awesome_webagent?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_webagent",
"locator": "",
"required_contents": {
"must_include": [
"awesome_webagent"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 480,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}}",
"instantiation_dict": {
"collaborator_account_list": "yjlou",
"repo": "solarized-prism-theme"
},
"intent": "Invite yjlou as collaborator to solarized-prism-theme",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/solarized-prism-theme/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"yjlou"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"gitlab"
],
"task_id": 481,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Abishek"
},
"intent": "Abishek wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 482,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "yjlou"
},
"intent": "yjlou wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 483,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Koushik"
},
"intent": "Koushik wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'koush')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 484,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Jakub Klinkovsk\u00fd"
},
"intent": "Jakub Klinkovsk\u00fd wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 485,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Vinta"
},
"intent": "Vinta wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'vinta')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"shopping_admin"
],
"task_id": 486,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "404 Not Found",
"heading": "Bruh bro you clicked the wrong page"
},
"intent": "Change the page title of \"404 Not Found\" page on my site to \"Bruh bro you clicked the wrong page\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/1/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Bruh bro you clicked the wrong page"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 487,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Enable Cookies",
"heading": "Cookie monster coming to your place"
},
"intent": "Change the page title of \"Enable Cookies\" page on my site to \"Cookie monster coming to your place\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/3/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Cookie monster coming to your place"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 488,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Home Page",
"heading": "This is the home page!! Leave here!!"
},
"intent": "Change the page title of \"Home Page\" page on my site to \"This is the home page!! Leave here!!\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/2/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "This is the home page!! Leave here!!"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 489,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Privacy Policy",
"heading": "No privacy policy is needed in this dystopian world"
},
"intent": "Change the page title of \"Privacy Policy\" page on my site to \"No privacy policy is needed in this dystopian world\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/4/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "No privacy policy is needed in this dystopian world"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 490,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "About us",
"heading": "Secret"
},
"intent": "Change the page title of \"About us\" page on my site to \"Secret\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/5/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Secret"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 491,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Sarah Miller",
"message": "the order is ready to be shipped soon!"
},
"intent": "Notify Sarah Miller in their most recent pending order with message \"the order is ready to be shipped soon!\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "System message: We cannot add order history."
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 492,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Jane Doe",
"message": "sorry we are out of stock, please reorder"
},
"intent": "Notify Jane Doe in their most recent pending order with message \"sorry we are out of stock, please reorder\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
"locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
"required_contents": {
"exact_match": "sorry we are out of stock, please reorder"
}
}
]
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 493,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Grace Nguyen",
"message": "sorry we are bankrupt, please contact our customer service for refund"
},
"intent": "Notify Grace Nguyen in their most recent pending order with message \"sorry we are bankrupt, please contact our customer service for refund\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/307/",
"locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
"required_contents": {
"exact_match": "sorry we are bankrupt, please contact our customer service for refund"
}
}
]
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 494,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Alex Thomas",
"message": "Yo, your order will be shipped soon!"
},
"intent": "Notify Alex Thomas in their most recent pending order with message \"Yo, your order will be shipped soon!\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/304/",
"locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
"required_contents": {
"exact_match": "Yo, your order will be shipped soon!"
}
}
]
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 495,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Lily Potter",
"message": "Thanks, your order is ready to be shipped!"
},
"intent": "Notify Lily Potter in their most recent pending order with message \"Thanks, your order is ready to be shipped!\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/303/",
"locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
"required_contents": {
"exact_match": "Thanks, your order is ready to be shipped!"
}
}
]
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 496,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
"instantiation_dict": {
"tracking": "8974568499",
"order": "299",
"service": "Federal Express"
},
"intent": "Update order #299 with the Federal Express tracking number 8974568499",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/299/active_tab/order_shipments/",
"locator": "",
"required_contents": {
"must_include": [
"Tracking number 8974568499 for Federal Express assigned"
]
}
}
]
},
"intent_template_id": 284
},
{
"sites": [
"shopping_admin"
],
"task_id": 497,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
"instantiation_dict": {
"tracking": "24353446464",
"order": "307",
"service": "DHL"
},
"intent": "Update order #307 with the DHL tracking number 24353446464",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/307/active_tab/order_shipments/",
"locator": "",
"required_contents": {
"must_include": [
"Tracking number 24353446464 for DHL assigned"
]
}
}
]
},
"intent_template_id": 284
},
{
"sites": [
"shopping_admin"
],
"task_id": 498,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
"instantiation_dict": {
"tracking": "55591023930",
"order": "306",
"service": "UPS"
},
"intent": "Update order #306 with the UPS tracking number 55591023930",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/306/active_tab/order_shipments/",
"locator": "",
"required_contents": {
"must_include": [
"Tracking number 55591023930 for United Parcel Service assigned"
]
}
}
]
},
"intent_template_id": 284
},
{
"sites": [
"shopping_admin"
],
"task_id": 499,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
"instantiation_dict": {
"tracking": "13849373987",
"order": "304",
"service": "USPS"
},
"intent": "Update order #304 with the USPS tracking number 13849373987",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/304/active_tab/order_shipments/",
"locator": "",
"required_contents": {
"must_include": [
"Tracking number 13849373987 for United States Postal Service assigned"
]
}
}
]
},
"intent_template_id": 284
},
{
"sites": [
"shopping_admin"
],
"task_id": 500,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
"instantiation_dict": {
"tracking": "239028439840",
"order": "301",
"service": "DHL"
},
"intent": "Update order #301 with the DHL tracking number 239028439840",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/301/active_tab/order_shipments/",
"locator": "",
"required_contents": {
"must_include": [
"Tracking number 239028439840 for DHL assigned"
]
}
}
]
},
"intent_template_id": 284
},
{
"sites": [
"shopping_admin"
],
"task_id": 501,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Make all {{product}} as out of stock",
"instantiation_dict": {
"product": "Taurus Elements Shell"
},
"intent": "Make all Taurus Elements Shell as out of stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/350/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "0"
}
}
]
},
"intent_template_id": 287
},
{
"sites": [
"shopping_admin"
],
"task_id": 502,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Make all {{product}} as out of stock",
"instantiation_dict": {
"product": "Gobi HeatTec Tee"
},
"intent": "Make all Gobi HeatTec Tee as out of stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/446/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "0"
}
}
]
},
"intent_template_id": 287
},
{
"sites": [
"shopping_admin"
],
"task_id": 503,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Make all {{product}} as out of stock",
"instantiation_dict": {
"product": "rocco gym tank"
},
"intent": "Make all rocco gym tank as out of stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/682/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "0"
}
}
]
},
"intent_template_id": 287
},
{
"sites": [
"shopping_admin"
],
"task_id": 504,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Make all {{product}} as out of stock",
"instantiation_dict": {
"product": "Selene yoga hoodie"
},
"intent": "Make all Selene yoga hoodie as out of stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1108/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "0"
}
}
]
},
"intent_template_id": 287
},
{
"sites": [
"shopping_admin"
],
"task_id": 505,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Make all {{product}} as out of stock",
"instantiation_dict": {
"product": "Aeno capri"
},
"intent": "Make all Aeno capri as out of stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1861/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "0"
}
}
]
},
"intent_template_id": 287
},
{
"sites": [
"shopping"
],
"task_id": 506,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
"instantiation_dict": {
"product_category": "meat substitute",
"dollar_value": "between 100 and 200"
},
"intent": "Buy the highest rated product from the meat substitute category within a budget between 100 and 200.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B01CTR3DLE"
]
}
}
]
},
"intent_template_id": 172
},
{
"sites": [
"shopping"
],
"task_id": 507,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
"instantiation_dict": {
"product_category": "Ceiling light",
"dollar_value": "above 1000"
},
"intent": "Buy the highest rated product from the Ceiling light category within a budget above 1000.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B07BVL3P1V"
]
}
}
]
},
"intent_template_id": 172
},
{
"sites": [
"shopping"
],
"task_id": 508,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
"instantiation_dict": {
"product_category": "NS switch pouch",
"dollar_value": "under 60"
},
"intent": "Buy the highest rated product from the NS switch pouch category within a budget under 60.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B07116LGP6"
]
}
}
]
},
"intent_template_id": 172
},
{
"sites": [
"shopping"
],
"task_id": 509,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the best rating product from \"{{category}}\" category with at least 5 reviews and the product is least expensive",
"instantiation_dict": {
"category": "Men's shoe"
},
"intent": "Buy the best rating product from \"Men's shoe\" category with at least 5 reviews and the product is least expensive",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B01J4MM3KO"
]
}
}
]
},
"intent_template_id": 216
},
{
"sites": [
"shopping"
],
"task_id": 510,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the best rating product from \"{{category}}\" category with at least 5 reviews and the product is least expensive",
"instantiation_dict": {
"category": "Home Audio Speaker"
},
"intent": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B002R5ABIW"
]
}
}
]
},
"intent_template_id": 216
},
{
"sites": [
"shopping"
],
"task_id": 511,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add a {{product}} to my wish list.",
"instantiation_dict": {
"product": "laundry detergent"
},
"intent": "Add a laundry detergent to my wish list.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"laundry",
"detergent"
]
}
}
]
},
"intent_template_id": 189
},
{
"sites": [
"shopping"
],
"task_id": 512,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add a {{product}} to my wish list.",
"instantiation_dict": {
"product": "toothpaste"
},
"intent": "Add a toothpaste to my wish list.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"toothpaste"
]
}
}
]
},
"intent_template_id": 189
},
{
"sites": [
"shopping"
],
"task_id": 513,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add a {{product}} to my wish list.",
"instantiation_dict": {
"product": "chair"
},
"intent": "Add a chair to my wish list.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"chair"
]
}
}
]
},
"intent_template_id": 189
},
{
"sites": [
"shopping"
],
"task_id": 514,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add a {{product}} to my wish list.",
"instantiation_dict": {
"product": "white desk"
},
"intent": "Add a white desk to my wish list.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"white",
"desk"
]
}
}
]
},
"intent_template_id": 189
},
{
"sites": [
"shopping"
],
"task_id": 515,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add a {{product}} to my wish list.",
"instantiation_dict": {
"product": "white computer desk"
},
"intent": "Add a white computer desk to my wish list.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"white",
"computer",
"desk"
]
}
}
]
},
"intent_template_id": 189
},
{
"sites": [
"shopping"
],
"task_id": 516,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/elmwood-inn-fine-teas-orange-vanilla-caffeine-free-fruit-infusion-16-ounce-pouch.html",
"geolocation": null,
"intent_template": "Add this product to my wishlist",
"instantiation_dict": {},
"intent": "Add this product to my wishlist",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Elmwood Inn Fine Teas, Orange Vanilla Caffeine-free Fruit Infusion, 16-Ounce Pouch"
]
}
}
]
},
"intent_template_id": 196
},
{
"sites": [
"shopping"
],
"task_id": 517,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/skinit-decal-gaming-skin-compatible-with-xbox-one-s-console-and-controller-bundle-officially-licensed-nfl-baltimore-ravens-design.html",
"geolocation": null,
"intent_template": "Add this product to my wishlist",
"instantiation_dict": {},
"intent": "Add this product to my wishlist",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Skinit Decal Gaming Skin Compatible with Xbox One S Console and Controller Bundle - Officially Licensed NFL Baltimore Ravens Design"
]
}
}
]
},
"intent_template_id": 196
},
{
"sites": [
"shopping"
],
"task_id": 518,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/sceptre-e195bd-srr-19-inch-720p-led-tv-true-black-2017.html",
"geolocation": null,
"intent_template": "Add this product to my wishlist",
"instantiation_dict": {},
"intent": "Add this product to my wishlist",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Sceptre E195BD-SRR 19-Inch 720P LED TV, True Black (2017)"
]
}
}
]
},
"intent_template_id": 196
},
{
"sites": [
"shopping"
],
"task_id": 519,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/iphone-13-pro-max-case-neon-turtle-iphone-13-pro-max-cases-tempered-glass-back-soft-silicone-tpu-shock-protective-case-for-apple-iphone-13-pro-max.html",
"geolocation": null,
"intent_template": "Add this product to my wishlist",
"instantiation_dict": {},
"intent": "Add this product to my wishlist",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"iPhone 13 Pro Max Case, Neon Turtle iPhone 13 Pro Max Cases, Tempered Glass Back+Soft Silicone TPU Shock Protective Case for Apple iPhone 13 Pro Max"
]
}
}
]
},
"intent_template_id": 196
},
{
"sites": [
"shopping"
],
"task_id": 520,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/magnetic-metal-stainless-steel-d-pads-kits-directional-pad-replacement-parts-for-xbox-one-elite-controller-elite-series-2-xbox-one-xbox-one-s-x-controller.html",
"geolocation": null,
"intent_template": "Add this product to my wishlist",
"instantiation_dict": {},
"intent": "Add this product to my wishlist",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Magnetic Metal Stainless Steel D-pads Kits Directional Pad Replacement Parts for Xbox One Elite Controller, Elite Series 2, Xbox One, Xbox One S/X Controller"
]
}
}
]
},
"intent_template_id": 196
},
{
"sites": [
"shopping"
],
"task_id": 521,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Subscribe to the newsletter of OneStopMarket",
"instantiation_dict": {},
"intent": "Subscribe to the newsletter of OneStopMarket",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/newsletter/manage/",
"locator": "document.querySelector('[title=\"General Subscription\"').checked.toString()",
"required_contents": {
"exact_match": "true"
}
}
]
},
"intent_template_id": 199
},
{
"sites": [
"gitlab"
],
"task_id": 522,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "all repos from facebook"
},
"intent": "Fork all repos from facebook.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/projects",
"locator": "document.querySelector('[data-qa-selector=\"projects_list\"').outerText",
"required_contents": {
"must_include": [
"create-react-app",
"buck"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 523,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Star the top {{number}} most stared repos in Gitlab",
"instantiation_dict": {
"number": "five"
},
"intent": "Star the top five most stared repos in Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/starred",
"locator": "",
"required_contents": {
"must_include": [
"AndroidSlidingUpPanel",
"create-react-app",
"ffmpeg-python",
"PHP_XLSXWriter",
"AndroidAsync"
]
}
}
]
},
"intent_template_id": 354
},
{
"sites": [
"gitlab"
],
"task_id": 524,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Star the top {{number}} most stared repos in Gitlab",
"instantiation_dict": {
"number": "eight"
},
"intent": "Star the top eight most stared repos in Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/starred",
"locator": "",
"required_contents": {
"must_include": [
"AndroidSlidingUpPanel",
"create-react-app",
"ffmpeg-python",
"PHP_XLSXWriter",
"AndroidAsync",
"Pytorch-GAN",
"administrate",
"keycloak"
]
}
}
]
},
"intent_template_id": 354
},
{
"sites": [
"gitlab"
],
"task_id": 525,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Star the top {{number}} most stared repos in Gitlab",
"instantiation_dict": {
"number": "four"
},
"intent": "Star the top four most stared repos in Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/starred",
"locator": "",
"required_contents": {
"must_include": [
"AndroidSlidingUpPanel",
"create-react-app",
"ffmpeg-python",
"PHP_XLSXWriter"
]
}
}
]
},
"intent_template_id": 354
},
{
"sites": [
"gitlab"
],
"task_id": 526,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Star the top {{number}} most stared repos in Gitlab",
"instantiation_dict": {
"number": "three"
},
"intent": "Star the top three most stared repos in Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/starred",
"locator": "",
"required_contents": {
"must_include": [
"AndroidSlidingUpPanel",
"create-react-app",
"ffmpeg-python"
]
}
}
]
},
"intent_template_id": 354
},
{
"sites": [
"gitlab"
],
"task_id": 527,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Star the top {{number}} most stared repos in Gitlab",
"instantiation_dict": {
"number": "one"
},
"intent": "Star the top one most stared repos in Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/starred",
"locator": "",
"required_contents": {
"must_include": [
"AndroidSlidingUpPanel"
]
}
}
]
},
"intent_template_id": 354
},
{
"sites": [
"shopping"
],
"task_id": 528,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"instantiation_dict": {
"product": "phone screen protector",
"time": "March 2023"
},
"intent": "Draft a refund message via their \"contact us\" form for the phone screen protector I bought March 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000180",
"12.99"
]
}
}
]
},
"intent_template_id": 154
},
{
"sites": [
"shopping"
],
"task_id": 529,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"instantiation_dict": {
"product": "bluetooth speaker",
"time": "Feb 2023"
},
"intent": "Draft a refund message via their \"contact us\" form for the bluetooth speaker I bought Feb 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000148",
"169.95"
]
}
}
]
},
"intent_template_id": 154
},
{
"sites": [
"shopping"
],
"task_id": 530,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"instantiation_dict": {
"product": "kitchen organizer",
"time": "around Feb 2023"
},
"intent": "Draft a refund message via their \"contact us\" form for the kitchen organizer I bought around Feb 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000161",
"68.88"
]
}
}
]
},
"intent_template_id": 154
},
{
"sites": [
"shopping"
],
"task_id": 531,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"instantiation_dict": {
"product": "phone case",
"time": "March 2023"
},
"intent": "Draft a refund message via their \"contact us\" form for the phone case I bought March 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000180",
"$12.99"
]
}
}
]
},
"intent_template_id": 154
},
{
"sites": [
"shopping"
],
"task_id": 532,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"instantiation_dict": {
"product": "PS3 remote controller",
"time": "early 2023"
},
"intent": "Draft a refund message via their \"contact us\" form for the PS3 remote controller I bought early 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000180",
"1.63"
]
}
}
]
},
"intent_template_id": 154
},
{
"sites": [
"gitlab"
],
"task_id": 533,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Follow {{account_list}} on Gitlab",
"instantiation_dict": {
"account_list": [
"convexegg",
"yjlou"
]
},
"intent": "Follow ['convexegg', 'yjlou'] on Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/following",
"locator": "document.querySelector('.user-profile').outerText",
"required_contents": {
"must_include": [
"@convexegg",
"@yjlou"
]
}
}
]
},
"intent_template_id": 330
},
{
"sites": [
"gitlab"
],
"task_id": 534,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Follow {{account_list}} on Gitlab",
"instantiation_dict": {
"account_list": [
"Jakub Klinkovsk\u00fd",
"Koushik",
"Vinta Chen"
]
},
"intent": "Follow ['Jakub Klinkovsk\u00fd', 'Koushik', 'Vinta Chen'] on Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/following",
"locator": "document.querySelector('.user-profile').outerText",
"required_contents": {
"must_include": [
"@lahwaacz",
"@koush",
"@vinta"
]
}
}
]
},
"intent_template_id": 330
},
{
"sites": [
"gitlab"
],
"task_id": 535,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Follow {{account_list}} on Gitlab",
"instantiation_dict": {
"account_list": [
"Jakub K",
"ghost",
"Beno\u00eet Blanchon"
]
},
"intent": "Follow ['Jakub K', 'ghost', 'Beno\u00eet Blanchon'] on Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/following",
"locator": "document.querySelector('.user-profile').outerText",
"required_contents": {
"must_include": [
"@lahwaacz",
"@ghost",
"@bblanchon"
]
}
}
]
},
"intent_template_id": 330
},
{
"sites": [
"gitlab"
],
"task_id": 536,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Follow {{account_list}} on Gitlab",
"instantiation_dict": {
"account_list": [
"ghost",
"R1kk3r",
"Abishek"
]
},
"intent": "Follow ['ghost', 'R1kk3r', 'Abishek'] on Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/following",
"locator": "document.querySelector('.user-profile').outerText",
"required_contents": {
"must_include": [
"@lahwaacz",
"@R1kk3r",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 330
},
{
"sites": [
"gitlab"
],
"task_id": 537,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Follow {{account_list}} on Gitlab",
"instantiation_dict": {
"account_list": [
"Jakub Klinkovsk",
"convexegg",
"Vinta Chen",
"yjlou",
"Abishek S"
]
},
"intent": "Follow ['Jakub Klinkovsk', 'convexegg', 'Vinta Chen', 'yjlou', 'Abishek S'] on Gitlab",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/users/byteblaze/following",
"locator": "document.querySelector('.user-profile').outerText",
"required_contents": {
"must_include": [
"@lahwaacz",
"@convexegg",
"@vinta",
"@yjlou",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 330
},
{
"sites": [
"shopping_admin"
],
"task_id": 538,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Modify the address of order #{{order_id}} to {{address}}",
"instantiation_dict": {
"order_id": "299",
"address": "456 Oak Avenue, Apartment 5B, New York, NY, 10001"
},
"intent": "Modify the address of order #299 to 456 Oak Avenue, Apartment 5B, New York, NY, 10001",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/299",
"locator": "",
"required_contents": {
"must_include": [
"456 Oak Avenue",
"Apartment 5B",
"New York",
"10001"
]
}
}
]
},
"intent_template_id": 240
},
{
"sites": [
"shopping_admin"
],
"task_id": 539,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Modify the address of order #{{order_id}} to {{address}}",
"instantiation_dict": {
"order_id": "65",
"address": "789 Pine Lane, San Francisco, CA, 94102"
},
"intent": "Modify the address of order #65 to 789 Pine Lane, San Francisco, CA, 94102",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/65",
"locator": "",
"required_contents": {
"must_include": [
"789 Pine Lane",
"San Francisco",
"California",
"94102"
]
}
}
]
},
"intent_template_id": 240
},
{
"sites": [
"shopping_admin"
],
"task_id": 540,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Modify the address of order #{{order_id}} to {{address}}",
"instantiation_dict": {
"order_id": "301",
"address": "321 Birch Boulevard, Suite 200, Dallas, TX, 75201"
},
"intent": "Modify the address of order #301 to 321 Birch Boulevard, Suite 200, Dallas, TX, 75201",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/301",
"locator": "",
"required_contents": {
"must_include": [
"321 Birch Boulevard",
"Suite 200",
"Dallas",
"Texas",
"75201"
]
}
}
]
},
"intent_template_id": 240
},
{
"sites": [
"shopping_admin"
],
"task_id": 541,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Modify the address of order #{{order_id}} to {{address}}",
"instantiation_dict": {
"order_id": "125",
"address": "654 Elm Drive, Apartment 12, Miami, FL, 33101"
},
"intent": "Modify the address of order #125 to 654 Elm Drive, Apartment 12, Miami, FL, 33101",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/125",
"locator": "",
"required_contents": {
"must_include": [
"654 Elm Drive",
"Apartment 12",
"Miami",
"Florida",
"33101"
]
}
}
]
},
"intent_template_id": 240
},
{
"sites": [
"shopping_admin"
],
"task_id": 542,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Modify the address of order #{{order_id}} to {{address}}",
"instantiation_dict": {
"order_id": "300",
"address": "987 Cedar Court, Los Angeles, CA, 90012"
},
"intent": "Modify the address of order #300 to 987 Cedar Court, Los Angeles, CA, 90012",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/300",
"locator": "",
"required_contents": {
"must_include": [
"987 Cedar Court",
"Los Angeles",
"California",
"90012"
]
}
}
]
},
"intent_template_id": 240
},
{
"sites": [
"shopping_admin"
],
"task_id": 543,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the product description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Bella Tank"
},
"intent": "Update the product description of Bella Tank to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../bella-tank.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"Good choice for working out and stylin' enough to wear when I'm hanging with friends on hot days",
"Also washes really well",
"Always a sweet n sporty look for the gym",
"Keeps me cool and the seams don't rub up against me like some of my other tanks"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping_admin"
],
"task_id": 544,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Selena Yoga Hoodie"
},
"intent": "Update the description of Selena Yoga Hoodie to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../selene-yoga-hoodie.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"I was super cold and it did the job.",
"The sleeves are definitely thicker than you realize, which is a good thing",
"really quite substantial",
"planning on buying another one of these in another color",
"the best hoodie ive ever owned"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping_admin"
],
"task_id": 545,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Radiant Tee"
},
"intent": "Update the description of Radiant Tee to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../radiant-tee.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"What I rally love here is that it does the job of keeping me cool and dry",
"I'm a big guy and sweat A LOT",
"Even after a day of gulf, I'm still dry and comfortable",
"What a versatile shirt",
"Not only does it feel very soft compared to my old worn out polos, but it also does the job promised",
"I like going out after my game for drinks so I look good then too and don't need to change into something fresh"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping_admin"
],
"task_id": 546,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Lucia Cross-Fit Bra"
},
"intent": "Update the description of Lucia Cross-Fit Bra to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../affirm-water-bottle.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"Wide mouth opening makes it easy to clean"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping_admin"
],
"task_id": 547,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a new {{option}} option {{value}} to the {{base_setting}} of {{product}}",
"instantiation_dict": {
"option": "color",
"value": "brown",
"base_setting": "size S",
"product": "Phoebe Zipper Sweatshirt"
},
"intent": "Add a new color option brown to the size S of Phoebe Zipper Sweatshirt",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1130/",
"locator": "document.querySelector('[data-index=\"configurable\"').outerText",
"required_contents": {
"must_include": [
"Phoebe Zipper Sweatshirt-S-Brown"
]
}
}
]
},
"intent_template_id": 252
},
{
"sites": [
"shopping_admin"
],
"task_id": 548,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a new {{option}} {{value}} to {{base_setting}} of {{product}}",
"instantiation_dict": {
"option": "color",
"value": "blue",
"base_setting": "size S and M",
"product": "Frankie Sweatshirt"
},
"intent": "Add a new color blue to size S and M of Frankie Sweatshirt",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/110/",
"locator": "document.querySelector('[data-index=\"configurable\"').outerText",
"required_contents": {
"must_include": [
"Sweatshirt-M-Blue",
"Sweatshirt-S-Blue"
]
}
}
]
},
"intent_template_id": 252
},
{
"sites": [
"shopping_admin"
],
"task_id": 549,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a new {{option}} {{value}} to {{base_setting}} {{product}}",
"instantiation_dict": {
"option": "size",
"value": "XXXL",
"base_setting": "green",
"product": "Minerva LumaTech V-Tee"
},
"intent": "Add a new size XXXL to green Minerva LumaTech V-Tee",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1492/",
"locator": "document.querySelector('[data-index=\"configurable\"').outerText",
"required_contents": {
"must_include": [
"V-Tee-XXXL-Green"
]
}
}
]
},
"intent_template_id": 252
},
{
"sites": [
"shopping_admin"
],
"task_id": 550,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a new {{option}} {{value}} to {{base_setting}} {{product}}",
"instantiation_dict": {
"option": "size",
"value": "XXS",
"base_setting": "blue and purple",
"product": "Nona Fitness Tank"
},
"intent": "Add a new size XXS to blue and purple Nona Fitness Tank",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1732/",
"locator": "document.querySelector('[data-index=\"configurable\"').outerText",
"required_contents": {
"must_include": [
"Tank-XXS-Blue",
"Tank-XXS-Purple"
]
}
}
]
},
"intent_template_id": 252
},
{
"sites": [
"shopping_admin"
],
"task_id": 551,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add new {{option}} {{value}} to {{base_setting}} of {{product}}",
"instantiation_dict": {
"option": "size",
"value": "30 and 31",
"base_setting": "all color variants",
"product": "Diana Tights"
},
"intent": "Add new size 30 and 31 to all color variants of Diana Tights",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1854/",
"locator": "document.querySelector('[data-index=\"configurable\"').outerText",
"required_contents": {
"must_include": [
"Tights-30-Blue",
"Tights-30-Black",
"Tights-30-Orange",
"Tights-31-Blue",
"Tights-31-Black",
"Tights-31-Orange"
]
}
}
]
},
"intent_template_id": 252
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 552,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
"instantiation_dict": {
"directory": "real_space",
"subreddit": "space",
"gitlab_repo": "gimmiethat.space"
},
"intent": "Create a folder named real_space in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the space?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/real_space/urls.txt",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/space/134164/scientists-erupt-at-nasa-gutting-funding-for-crucial-venus",
"__REDDIT__/f/space/134163/virgin-orbit-fails-to-secure-funding-will-cease-operations",
"__REDDIT__/f/space/134162/nasa-to-name-artemis-2-crew-next-week-the-first-moon",
"__REDDIT__/f/space/134161/bent-light-in-deep-space-reveals-one-of-the-biggest-black",
"__REDDIT__/f/space/134160/seti-s-new-machine-learning-algorithm-works-like-google-s"
]
}
}
]
},
"intent_template_id": 84
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 553,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
"instantiation_dict": {
"directory": "news",
"gitlab_repo": "gimmiethat.space",
"subreddit": "news related subreddits"
},
"intent": "Create a folder named news in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the news related subreddits?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/news/urls.txt",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/news/129905/ohio-man-charged-for-using-molotov-cocktails-to-attack",
"__REDDIT__/f/news/129904/in-a-loss-for-fox-news-judge-allows-dominion-s-defamation",
"__REDDIT__/f/news/129903/theater-group-sues-to-block-tennessee-s-new-anti-drag-law",
"__REDDIT__/f/news/129902/andrew-tate-released-from-jail-in-romania-and-placed-under",
"__REDDIT__/f/news/129901/rare-high-risk-storm-alert-issued-for-parts-of-midwest-and"
]
}
}
]
},
"intent_template_id": 84
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 554,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
"instantiation_dict": {
"directory": "moive_space",
"gitlab_repo": "gimmiethat.space",
"subreddit": "movies"
},
"intent": "Create a folder named moive_space in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the movies?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/moive_space/urls.txt",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/movies/128825/scenes-in-film-that-feel-off-or-wrong-in-some-way-and-make",
"__REDDIT__/f/movies/128824/disney-s-live-action-lilo-amp-stitch-movie-finds-its-lilo-in",
"__REDDIT__/f/movies/128823/fantastic-four-movie-gets-new-writer-with-avatar-the-way-of",
"__REDDIT__/f/movies/128822/can-someone-explain-what-made-steven-seagal-so-appealing-for",
"__REDDIT__/f/movies/128821/ban-on-fetish-sex-depictions-in-film-should-end-australia"
]
}
}
]
},
"intent_template_id": 84
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 555,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
"instantiation_dict": {
"directory": "funny_pic",
"gitlab_repo": "gimmiethat.space",
"subreddit": "memes"
},
"intent": "Create a folder named funny_pic in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the memes?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/funny_pic/urls.txt",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/memes/127991/it-do-be-like-that-tho",
"__REDDIT__/f/memes/127990/thank-you-memers-this-wouldn-t-be-possible-without-you",
"__REDDIT__/f/memes/127989/if-you-have-no-other-choice",
"__REDDIT__/f/memes/127988/yes-yes-yes",
"__REDDIT__/f/memes/127987/shagadelic-baby"
]
}
}
]
},
"intent_template_id": 84
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 556,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "nolan_honest_fans",
"topics": "movies directed by Christopher Nolan"
},
"intent": "Create a repo named nolan_honest_fans with movies directed by Christopher Nolan in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/nolan_honest_fans/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"Following",
"Memento",
"Insomnia",
"Batman Begins",
"The Prestige",
"The Dark Knight",
"Inception",
"The Dark Knight Rises",
"Interstellar",
"Dunkirk",
"Tenet",
"Oppenheimer"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 557,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "nolan_old_fans",
"topics": "movies directed by Christopher Nolan before 2010"
},
"intent": "Create a repo named nolan_old_fans with movies directed by Christopher Nolan before 2010 in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/nolan_old_fans/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"Following",
"Memento",
"Insomnia",
"Batman Begins",
"The Prestige",
"The Dark Knight"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 558,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "nolan_young_fans",
"topics": "movies directed by Christopher Nolan after 2010"
},
"intent": "Create a repo named nolan_young_fans with movies directed by Christopher Nolan after 2010 in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/nolan_young_fans/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"Inception",
"The Dark Knight Rises",
"Interstellar",
"Dunkirk",
"Tenet",
"Oppenheimer"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 559,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "nolan_followers",
"topics": "career timeline of Christopher Nolan"
},
"intent": "Create a repo named nolan_followers with career timeline of Christopher Nolan in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/nolan_followers/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"1993\u20132003: Early career and breakthrough",
"2003\u20132013: Widespread recognition",
"2014\u20132019: Established Hollywood auteur",
"2020\u2013present"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 560,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "nolan_academy_awards",
"topics": "movies that won Academy Awards by Christopher Nolan"
},
"intent": "Create a repo named nolan_academy_awards with movies that won Academy Awards by Christopher Nolan in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/nolan_academy_awards/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"The Dark Knight",
"Inception",
"Interstellar",
"Dunkirk",
"Tenet"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"wikipedia"
],
"task_id": 561,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
"instantiation_dict": {
"name": "bafta_awards_nolan",
"topics": "movies that are nominated BAFTA Awards by Christopher Nolan"
},
"intent": "Create a repo named bafta_awards_nolan with movies that are nominated BAFTA Awards by Christopher Nolan in a README file",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/bafta_awards_nolan/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"Batman Begins",
"The Dark Knight",
"Inception",
"The Dark Knight Rises",
"Interstellar",
"Dunkirk",
"Tenet"
]
}
}
]
},
"intent_template_id": 87
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 562,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
"instantiation_dict": {
"name": "Awesome_DIY_ideas",
"num": 6
},
"intent": "create a repository named Awesome_DIY_ideas that includes a README file with the links to the most active 6 DIY ideas on DIY subreddit?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/Awesome_DIY_ideas/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
"__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
"__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
"__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
"__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
"__REDDIT__/f/DIY/118931/afci-outlet-question"
]
}
}
]
},
"intent_template_id": 88
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 563,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
"instantiation_dict": {
"name": "fun_thing_to_do",
"num": 5
},
"intent": "create a repository named fun_thing_to_do that includes a README file with the links to the most active 5 DIY ideas on DIY subreddit?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/fun_thing_to_do/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
"__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
"__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
"__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
"__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches"
]
}
}
]
},
"intent_template_id": 88
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 564,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
"instantiation_dict": {
"name": "live_a_life",
"num": 3
},
"intent": "create a repository named live_a_life that includes a README file with the links to the most active 3 DIY ideas on DIY subreddit?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/live_a_life/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
"__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
"__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing"
]
}
}
]
},
"intent_template_id": 88
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 565,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
"instantiation_dict": {
"name": "TODO",
"num": 10
},
"intent": "create a repository named TODO that includes a README file with the links to the most active 10 DIY ideas on DIY subreddit?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/TODO/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
"__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
"__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
"__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
"__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
"__REDDIT__/f/DIY/118931/afci-outlet-question",
"__REDDIT__/f/DIY/118824/teflon-tape-to-attach-washing-machine-drain-hose-to-pipe",
"__REDDIT__/f/DIY/118866/paver-base-for-shed",
"__REDDIT__/f/DIY/118820/ways-to-locate-our-buried-electrical-service",
"__REDDIT__/f/DIY/118836/how-to-eliminate-transitions-for-disability-mobility-reasons"
]
}
}
]
},
"intent_template_id": 88
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 566,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
"instantiation_dict": {
"name": "Do it myself",
"num": 8
},
"intent": "create a repository named Do it myself that includes a README file with the links to the most active 8 DIY ideas on DIY subreddit?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/Do-it-myself/-/raw/main/README.md",
"locator": "",
"required_contents": {
"must_include": [
"__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
"__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
"__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
"__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
"__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
"__REDDIT__/f/DIY/118931/afci-outlet-question",
"__REDDIT__/f/DIY/118824/teflon-tape-to-attach-washing-machine-drain-hose-to-pipe",
"__REDDIT__/f/DIY/118866/paver-base-for-shed"
]
}
}
]
},
"intent_template_id": 88
},
{
"sites": [
"gitlab"
],
"task_id": 567,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
"instantiation_dict": {
"collaborator_account_list": "Jakub Klinkovsk\u00fd and Beno\u00eet Blanchon",
"repo": "gimmiethat.space"
},
"intent": "Invite Jakub Klinkovsk\u00fd and Beno\u00eet Blanchon as collaborator to gimmiethat.space repo",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@lahwaacz",
"@bblanchon"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"gitlab"
],
"task_id": 568,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
"instantiation_dict": {
"collaborator_account_list": "Abishek and Vinta",
"repo": "a11yproject.com"
},
"intent": "Invite Abishek and Vinta as collaborator to a11yproject.com repo",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@abisubramanya27",
"@vinta"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"gitlab"
],
"task_id": 569,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
"instantiation_dict": {
"collaborator_account_list": "Beno\u00eet and Abishek",
"repo": "my HTML5 markup extention"
},
"intent": "Invite Beno\u00eet and Abishek as collaborator to my HTML5 markup extention repo",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/accessible-html-content-patterns/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@bblanchon",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"gitlab"
],
"task_id": 570,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
"instantiation_dict": {
"collaborator_account_list": "Jakub K, Alex Dills, Alex Hutnik and Beno\u00eet Blanchon",
"repo": "my time tracking tool project"
},
"intent": "Invite Jakub K, Alex Dills, Alex Hutnik and Beno\u00eet Blanchon as collaborator to my time tracking tool project repo",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/timeit/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@lahwaacz",
"@V13Axel",
"@alexhutnik",
"@bblanchon"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"shopping"
],
"task_id": 571,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
"instantiation_dict": {
"address": "231 Willow Way, Suite 100, Chicago, IL, 60601"
},
"intent": "I recently moved, my address is 231 Willow Way, Suite 100, Chicago, IL, 60601, update my information on OneStopShopping accordingly",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
"required_contents": {
"must_include": [
"231 Willow Way",
"Suite 100",
"Chicago, Illinois, 60601"
]
}
},
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
"required_contents": {
"must_include": [
"231 Willow Way",
"Suite 100",
"Chicago, Illinois, 60601"
]
}
}
]
},
"intent_template_id": 165
},
{
"sites": [
"shopping"
],
"task_id": 572,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
"instantiation_dict": {
"address": "654 Aspen Road, House #3, Boston, MA, 02110"
},
"intent": "I recently moved, my address is 654 Aspen Road, House #3, Boston, MA, 02110, update my information on OneStopShopping accordingly",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
"required_contents": {
"must_include": [
"654 Aspen Road",
"House #3",
"Boston, Massachusetts, 02110"
]
}
},
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
"required_contents": {
"must_include": [
"654 Aspen Road",
"House #3",
"Boston, Massachusetts, 02110"
]
}
}
]
},
"intent_template_id": 165
},
{
"sites": [
"shopping"
],
"task_id": 573,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
"instantiation_dict": {
"address": "987 Sycamore Circle, Philadelphia, PA, 19102"
},
"intent": "I recently moved, my address is 987 Sycamore Circle, Philadelphia, PA, 19102, update my information on OneStopShopping accordingly",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
"required_contents": {
"must_include": [
"987 Sycamore Circle",
"Philadelphia, Pennsylvania, 19102"
]
}
},
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
"required_contents": {
"must_include": [
"987 Sycamore Circle",
"Philadelphia, Pennsylvania, 19102"
]
}
}
]
},
"intent_template_id": 165
},
{
"sites": [
"shopping"
],
"task_id": 574,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
"instantiation_dict": {
"address": "111 Magnolia Path, Atlanta, GA, 30303"
},
"intent": "I recently moved, my address is 111 Magnolia Path, Atlanta, GA, 30303, update my information on OneStopShopping accordingly",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
"required_contents": {
"must_include": [
"111 Magnolia Path",
"Atlanta, Georgia, 30303"
]
}
},
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
"required_contents": {
"must_include": [
"111 Magnolia Path",
"Atlanta, Georgia, 30303"
]
}
}
]
},
"intent_template_id": 165
},
{
"sites": [
"shopping"
],
"task_id": 575,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
"instantiation_dict": {
"address": "222 Redwood Rise, Suite 300, Seattle, WA, 98101"
},
"intent": "I recently moved, my address is 222 Redwood Rise, Suite 300, Seattle, WA, 98101, update my information on OneStopShopping accordingly",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
"required_contents": {
"must_include": [
"222 Redwood Rise",
"Suite 300",
"Seattle, Washington, 98101"
]
}
},
{
"url": "__SHOPPING__/customer/address",
"locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
"required_contents": {
"must_include": [
"222 Redwood Rise",
"Suite 300",
"Seattle, Washington, 98101"
]
}
}
]
},
"intent_template_id": 165
},
{
"sites": [
"gitlab"
],
"task_id": 576,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
"instantiation_dict": {
"repo": "a11y-webring.club",
"role": "developer",
"user_list": [
"abisubramanya27",
"lahwaacz"
]
},
"intent": "Add the following users to repo a11y-webring.club as developer: ['abisubramanya27', 'lahwaacz']",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/a11y-webring.club/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
"required_contents": {
"must_include": [
"Developer"
]
}
},
{
"url": "__GITLAB__/byteblaze/a11y-webring.club/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
"required_contents": {
"must_include": [
"Developer"
]
}
}
]
},
"intent_template_id": 351
},
{
"sites": [
"gitlab"
],
"task_id": 577,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Add the following users to my {{repo}} as {{role}}: {{user_list}}",
"instantiation_dict": {
"repo": "GitHub timeline item management extension",
"role": "maintainer",
"user_list": [
"abisubramanya27",
"lahwaacz"
]
},
"intent": "Add the following users to my GitHub timeline item management extension as maintainer: ['abisubramanya27', 'lahwaacz']",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/remove-board-movement-events-from-the-github-issue-timeline/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
"required_contents": {
"must_include": [
"Maintainer"
]
}
},
{
"url": "__GITLAB__/byteblaze/remove-board-movement-events-from-the-github-issue-timeline/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
"required_contents": {
"must_include": [
"Maintainer"
]
}
}
]
},
"intent_template_id": 351
},
{
"sites": [
"gitlab"
],
"task_id": 578,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
"instantiation_dict": {
"repo": "millennials-to-snake-people",
"role": "reporter",
"user_list": [
"yjlou",
"a11yproject"
]
},
"intent": "Add the following users to repo millennials-to-snake-people as reporter: ['yjlou', 'a11yproject']",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/millennials-to-snake-people/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
"required_contents": {
"must_include": [
"Reporter"
]
}
},
{
"url": "__GITLAB__/byteblaze/millennials-to-snake-people/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'a11yproject')",
"required_contents": {
"must_include": [
"Reporter"
]
}
}
]
},
"intent_template_id": 351
},
{
"sites": [
"gitlab"
],
"task_id": 579,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Add the following users to my {{repo}} as {{role}}: {{user_list}}",
"instantiation_dict": {
"repo": "time tracking tool",
"role": "guest",
"user_list": [
"yjlou"
]
},
"intent": "Add the following users to my time tracking tool as guest: ['yjlou']",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/timeit/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 351
},
{
"sites": [
"reddit"
],
"task_id": 580,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
"instantiation_dict": {
"name": "sci_fi",
"description": "A wild place for sci-fi enthusiasts",
"sidebar_list": [
"New",
"Classic",
"Movies",
"Post my novel",
"Random"
]
},
"intent": "Create a new forum named sci_fi, with a description of A wild place for sci-fi enthusiasts, and include ['New', 'Classic', 'Movies', 'Post my novel', 'Random'] in the sidebar?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/sci_fi/edit",
"locator": "document.querySelector(\"#forum_description\").value",
"required_contents": {
"must_include": [
"A wild place for sci-fi enthusiasts"
]
}
},
{
"url": "__REDDIT__/f/sci_fi/edit",
"locator": "document.querySelector(\"#forum_sidebar\").value",
"required_contents": {
"must_include": [
"New",
"Classic",
"Movies",
"Post my novel",
"Random"
]
}
}
]
},
"intent_template_id": 7
},
{
"sites": [
"reddit"
],
"task_id": 581,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
"instantiation_dict": {
"name": "cmu_lti",
"description": "Language Technologies Institute at Carnegie Mellon University",
"sidebar_list": [
"announcement",
"paper",
"alumni"
]
},
"intent": "Create a new forum named cmu_lti, with a description of Language Technologies Institute at Carnegie Mellon University, and include ['announcement', 'paper', 'alumni'] in the sidebar?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/cmu_lti/edit",
"locator": "document.querySelector(\"#forum_description\").value",
"required_contents": {
"must_include": [
"Language Technologies Institute at Carnegie Mellon University"
]
}
},
{
"url": "__REDDIT__/f/cmu_lti/edit",
"locator": "document.querySelector(\"#forum_sidebar\").value",
"required_contents": {
"must_include": [
"announcement",
"paper",
"alumni"
]
}
}
]
},
"intent_template_id": 7
},
{
"sites": [
"reddit"
],
"task_id": 582,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
"instantiation_dict": {
"name": "Cyberpunk",
"description": "Welcome to the future",
"sidebar_list": [
"Games",
"Books",
"Movies",
"Future"
]
},
"intent": "Create a new forum named Cyberpunk, with a description of Welcome to the future, and include ['Games', 'Books', 'Movies', 'Future'] in the sidebar?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/Cyberpunk/edit",
"locator": "document.querySelector(\"#forum_description\").value",
"required_contents": {
"must_include": [
"Welcome to the future"
]
}
},
{
"url": "__REDDIT__/f/Cyberpunk/edit",
"locator": "document.querySelector(\"#forum_sidebar\").value",
"required_contents": {
"must_include": [
"Games",
"Books",
"Movies",
"Future"
]
}
}
]
},
"intent_template_id": 7
},
{
"sites": [
"reddit"
],
"task_id": 583,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
"instantiation_dict": {
"name": "PlantsForCatParents",
"description": "Cat parents & plant lovers",
"sidebar_list": [
"Cat friendly",
"Local vendors",
"Promotion",
"Toxic plants!"
]
},
"intent": "Create a new forum named PlantsForCatParents, with a description of Cat parents & plant lovers, and include ['Cat friendly', 'Local vendors', 'Promotion', 'Toxic plants!'] in the sidebar?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/PlantsForCatParents/edit",
"locator": "document.querySelector(\"#forum_description\").value",
"required_contents": {
"must_include": [
"Cat parents & plant lovers"
]
}
},
{
"url": "__REDDIT__/f/PlantsForCatParents/edit",
"locator": "document.querySelector(\"#forum_sidebar\").value",
"required_contents": {
"must_include": [
"Cat friendly",
"Local vendors",
"Promotion",
"Toxic plants!"
]
}
}
]
},
"intent_template_id": 7
},
{
"sites": [
"reddit"
],
"task_id": 584,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
"instantiation_dict": {
"name": "Karaoke",
"description": "Place for Karaoke lovers",
"sidebar_list": [
"devices",
"setup"
]
},
"intent": "Create a new forum named Karaoke, with a description of Place for Karaoke lovers, and include ['devices', 'setup'] in the sidebar?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/Karaoke/edit",
"locator": "document.querySelector(\"#forum_description\").value",
"required_contents": {
"must_include": [
"Place for Karaoke lovers"
]
}
},
{
"url": "__REDDIT__/f/Karaoke/edit",
"locator": "document.querySelector(\"#forum_sidebar\").value",
"required_contents": {
"must_include": [
"devices",
"setup"
]
}
}
]
},
"intent_template_id": 7
},
{
"sites": [
"shopping"
],
"task_id": 585,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
"instantiation_dict": {
"product": "floor lamp",
"num_star": 5,
"nickname": "Emma Lopez"
},
"intent": "Rate my recent purchase of floor lamp with 5 stars, using my nickname Emma Lopez?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_rating('B00J8RZL7I')",
"required_contents": {
"must_include": [
"100"
]
}
},
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_author('B00J8RZL7I')",
"required_contents": {
"must_include": [
"Emma Lopez"
]
}
}
]
},
"intent_template_id": 194
},
{
"sites": [
"shopping"
],
"task_id": 586,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
"instantiation_dict": {
"product": "Jiffy Corn Muffin Cornbread Mix",
"num_star": 4,
"nickname": "ShoppingEmma"
},
"intent": "Rate my recent purchase of Jiffy Corn Muffin Cornbread Mix with 4 stars, using my nickname ShoppingEmma?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_rating('B07HZB38XH')",
"required_contents": {
"must_include": [
"80"
]
}
},
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_author('B07HZB38XH')",
"required_contents": {
"must_include": [
"ShoppingEmma"
]
}
}
]
},
"intent_template_id": 194
},
{
"sites": [
"shopping"
],
"task_id": 587,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
"instantiation_dict": {
"product": "PS3 Remote Controllers",
"num_star": 3,
"nickname": "GamingEmma"
},
"intent": "Rate my recent purchase of PS3 Remote Controllers with 3 stars, using my nickname GamingEmma?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_rating('B0041MSF2S')",
"required_contents": {
"must_include": [
"60"
]
}
},
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_author('B0041MSF2S')",
"required_contents": {
"must_include": [
"GamingEmma"
]
}
}
]
},
"intent_template_id": 194
},
{
"sites": [
"shopping"
],
"task_id": 588,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
"instantiation_dict": {
"product": "Foundation For Mattress With Frame Set",
"num_star": 1,
"nickname": "ShoppingEmma"
},
"intent": "Rate my recent purchase of Foundation For Mattress With Frame Set with 1 stars, using my nickname ShoppingEmma?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_rating('B07DFJ5XKH')",
"required_contents": {
"must_include": [
"20"
]
}
},
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_author('B07DFJ5XKH')",
"required_contents": {
"must_include": [
"ShoppingEmma"
]
}
}
]
},
"intent_template_id": 194
},
{
"sites": [
"shopping"
],
"task_id": 589,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
"instantiation_dict": {
"product": "Mini Wireless Bluetooth Speaker",
"num_star": 2,
"nickname": "SimpleEmma"
},
"intent": "Rate my recent purchase of Mini Wireless Bluetooth Speaker with 2 stars, using my nickname SimpleEmma?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_rating('B09P7BFL4H')",
"required_contents": {
"must_include": [
"40"
]
}
},
{
"url": "last",
"locator": "func:shopping_get_sku_latest_review_author('B09P7BFL4H')",
"required_contents": {
"must_include": [
"SimpleEmma"
]
}
}
]
},
"intent_template_id": 194
},
{
"sites": [
"gitlab"
],
"task_id": 590,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
"instantiation_dict": {
"event": "event of product launch",
"start_date": "1/16/2023",
"end_date": "1/30/2023"
},
"intent": "Create a milestone for the upcoming event of product launch starting on 1/16/2023 and ending on 1/30/2023",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/milestones",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"#content-body\").outerText",
"required_contents": {
"must_include": [
"product launch"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.start_date').outerText",
"required_contents": {
"must_include": [
"Jan 16, 2030"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.due_date').outerText",
"required_contents": {
"must_include": [
"Jan 30, 2030"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 339
},
{
"sites": [
"gitlab"
],
"task_id": 591,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
"instantiation_dict": {
"event": "practice of collective code review",
"start_date": "1/16/2023",
"end_date": "in 20 days"
},
"intent": "Create a milestone for the upcoming practice of collective code review starting on 1/16/2023 and ending on in 20 days",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/milestones",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"#content-body\").outerText",
"required_contents": {
"must_include": [
"code review"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.start_date').outerText",
"required_contents": {
"must_include": [
"Jan 16, 2030"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.due_date').outerText",
"required_contents": {
"must_include": [
"Feb 5, 2030"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 339
},
{
"sites": [
"gitlab"
],
"task_id": 592,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
"instantiation_dict": {
"event": "task of cleaning sensitive information",
"start_date": "2/16/2023",
"end_date": "in 20 days"
},
"intent": "Create a milestone for the upcoming task of cleaning sensitive information starting on 2/16/2023 and ending on in 20 days",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/milestones",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"#content-body\").outerText",
"required_contents": {
"must_include": [
"sensitive information"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.start_date').outerText",
"required_contents": {
"must_include": [
"Feb 16, 2030"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.due_date').outerText",
"required_contents": {
"must_include": [
"Mar 8, 2030"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 339
},
{
"sites": [
"gitlab"
],
"task_id": 593,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/dotfiles",
"geolocation": null,
"intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
"instantiation_dict": {
"event": "task of merging all branches to main",
"start_date": "March 15, 2044",
"end_date": "March 30, 2044"
},
"intent": "Create a milestone for the upcoming task of merging all branches to main starting on March 15, 2044 and ending on March 30, 2044",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/dotfiles/-/milestones",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"#content-body\").outerText",
"required_contents": {
"must_include": [
"all branches to main"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.start_date').outerText",
"required_contents": {
"must_include": [
"Mar 15, 2044"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.due_date').outerText",
"required_contents": {
"must_include": [
"Mar 30, 2044"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 339
},
{
"sites": [
"gitlab"
],
"task_id": 594,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/dotfiles",
"geolocation": null,
"intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
"instantiation_dict": {
"event": "task of adding a new branch for zsh comprehensive support",
"start_date": "5/1/2044",
"end_date": "in 20 days"
},
"intent": "Create a milestone for the upcoming task of adding a new branch for zsh comprehensive support starting on 5/1/2044 and ending on in 20 days",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/dotfiles/-/milestones",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"#content-body\").outerText",
"required_contents": {
"must_include": [
"zsh comprehensive support"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.start_date').outerText",
"required_contents": {
"must_include": [
"May 1, 2044"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.due_date').outerText",
"required_contents": {
"must_include": [
"May 21, 2044"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 339
},
{
"sites": [
"reddit"
],
"task_id": 595,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
"instantiation_dict": {
"subreddit": "space"
},
"intent": "Open the thread of a trending post on the forum \"space\" and subscribe.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/space",
"program_html": [
{
"url": "__REDDIT__/",
"locator": "document.querySelector(\"#sidebar > section\").outerText",
"required_contents": {
"must_include": [
"space"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 4
},
{
"sites": [
"reddit"
],
"task_id": 596,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
"instantiation_dict": {
"subreddit": "books"
},
"intent": "Open the thread of a trending post on the forum \"books\" and subscribe.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "__REDDIT__/",
"locator": "document.querySelector(\"#sidebar > section\").outerText",
"required_contents": {
"must_include": [
"books"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 4
},
{
"sites": [
"reddit"
],
"task_id": 597,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
"instantiation_dict": {
"subreddit": "consoles"
},
"intent": "Open the thread of a trending post on the forum \"consoles\" and subscribe.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/consoles",
"program_html": [
{
"url": "__REDDIT__/",
"locator": "document.querySelector(\"#sidebar > section\").outerText",
"required_contents": {
"must_include": [
"consoles"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 4
},
{
"sites": [
"reddit"
],
"task_id": 598,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
"instantiation_dict": {
"subreddit": "pittsburgh"
},
"intent": "Open the thread of a trending post on the forum \"pittsburgh\" and subscribe.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/pittsburgh",
"program_html": [
{
"url": "__REDDIT__/",
"locator": "document.querySelector(\"#sidebar > section\").outerText",
"required_contents": {
"must_include": [
"pittsburgh"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 4
},
{
"sites": [
"reddit"
],
"task_id": 599,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
"instantiation_dict": {
"subreddit": "machine learning"
},
"intent": "Open the thread of a trending post on the forum \"machine learning\" and subscribe.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/MachineLearning",
"program_html": [
{
"url": "__REDDIT__/",
"locator": "document.querySelector(\"#sidebar > section\").outerText",
"required_contents": {
"must_include": [
"MachineLearning"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 4
},
{
"sites": [
"reddit"
],
"task_id": 600,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
"instantiation_dict": {
"question": "what is the recommended console to buy these days"
},
"intent": "Post my question, \"what is the recommended console to buy these days\", in a subreddit where I'm likely to get an answer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/consoles",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"what is the recommended console to buy these days"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 3765
},
{
"sites": [
"reddit"
],
"task_id": 601,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
"instantiation_dict": {
"question": "is car necessary in NYC"
},
"intent": "Post my question, \"is car necessary in NYC\", in a subreddit where I'm likely to get an answer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/nyc",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"is car necessary in NYC |OR| is a car necessary in NYC"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 3765
},
{
"sites": [
"reddit"
],
"task_id": 602,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
"instantiation_dict": {
"question": "places for new drivers to learn driving in pittsburgh"
},
"intent": "Post my question, \"places for new drivers to learn driving in pittsburgh\", in a subreddit where I'm likely to get an answer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/pittsburgh",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"places for new drivers to learn driving in pittsburgh"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 3765
},
{
"sites": [
"reddit"
],
"task_id": 603,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
"instantiation_dict": {
"question": "safe and budget apartment to live in nyc"
},
"intent": "Post my question, \"safe and budget apartment to live in nyc\", in a subreddit where I'm likely to get an answer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/nyc",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"safe and budget apartment to live in nyc"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 3765
},
{
"sites": [
"reddit"
],
"task_id": 604,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
"instantiation_dict": {
"question": "what is the SOTA web navigation agent repo"
},
"intent": "Post my question, \"what is the SOTA web navigation agent repo\", in a subreddit where I'm likely to get an answer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/deeplearning |OR| __REDDIT__/f/MachineLearning |OR| __REDDIT__/f/singularity",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"what is the SOTA web navigation agent repo"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 3765
},
{
"sites": [
"reddit"
],
"task_id": 605,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
"instantiation_dict": {
"topic": "gaming consoles",
"question": "what is the recommended console to buy these days"
},
"intent": "Find a subreddit focused on topics related to gaming consoles, and post my question, \"what is the recommended console to buy these days\" there",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/consoles",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"what is the recommended console to buy these days"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 5
},
{
"sites": [
"reddit"
],
"task_id": 606,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
"instantiation_dict": {
"topic": "NYC",
"question": "is car necessary"
},
"intent": "Find a subreddit focused on topics related to NYC, and post my question, \"is car necessary\" there",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/nyc",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"is car necessary |OR| is a car necessary"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 5
},
{
"sites": [
"reddit"
],
"task_id": 607,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
"instantiation_dict": {
"topic": "city Pittsburgh",
"question": "places for new drivers to learn driving"
},
"intent": "Find a subreddit focused on topics related to city Pittsburgh, and post my question, \"places for new drivers to learn driving\" there",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/pittsburgh",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"places for new drivers to learn driving"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 5
},
{
"sites": [
"reddit"
],
"task_id": 608,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
"instantiation_dict": {
"topic": "city lives in DMV area",
"question": "safe and budget apartment to live"
},
"intent": "Find a subreddit focused on topics related to city lives in DMV area, and post my question, \"safe and budget apartment to live\" there",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/washington |OR| __REDDIT__/f/washingtondc",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"safe and budget apartment to live"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 5
},
{
"sites": [
"reddit"
],
"task_id": 609,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
"instantiation_dict": {
"topic": "ML, DL, NLP",
"question": "what is the SOTA web navigation agent repo"
},
"intent": "Find a subreddit focused on topics related to ML, DL, NLP, and post my question, \"what is the SOTA web navigation agent repo\" there",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/deeplearning |OR| __REDDIT__/f/MachineLearning |OR| __REDDIT__/f/singularity",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"what is the SOTA web navigation agent repo"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 5
},
{
"sites": [
"reddit"
],
"task_id": 610,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
"instantiation_dict": {
"book": "To Kill a Mockingbird by Harper Lee",
"content": "good book!"
},
"intent": "Post a review of my recent reading \"To Kill a Mockingbird by Harper Lee\" in the r/books with my comment \"good book!\".",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"To Kill a Mockingbird by Harper Lee",
"good book!"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 9
},
{
"sites": [
"reddit"
],
"task_id": 611,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
"instantiation_dict": {
"book": "Harry Potter",
"content": "Wonderful journey"
},
"intent": "Post a review of my recent reading \"Harry Potter\" in the r/books with my comment \"Wonderful journey\".",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Harry Potter",
"Wonderful journey"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 9
},
{
"sites": [
"reddit"
],
"task_id": 612,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
"instantiation_dict": {
"book": "big little lies",
"content": "can't stop it"
},
"intent": "Post a review of my recent reading \"big little lies\" in the r/books with my comment \"can't stop it\".",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"big little lies",
"can't stop it"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 9
},
{
"sites": [
"reddit"
],
"task_id": 613,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
"instantiation_dict": {
"book": "Love story",
"content": "I cried"
},
"intent": "Post a review of my recent reading \"Love story\" in the r/books with my comment \"I cried\".",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Love story",
"I cried"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 9
},
{
"sites": [
"reddit"
],
"task_id": 614,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
"instantiation_dict": {
"book": "Gone with the wind",
"content": "It's a book with history"
},
"intent": "Post a review of my recent reading \"Gone with the wind\" in the r/books with my comment \"It's a book with history\".",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Gone with the wind",
"It's a book with history"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 9
},
{
"sites": [
"reddit"
],
"task_id": 615,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pics",
"geolocation": null,
"intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
"instantiation_dict": {
"content": "Bald Eagle",
"subreddit": "earthporn"
},
"intent": "Re-post the image of Bald Eagle in this page to earthporn subreddit and note \"from /f/pics\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/earthporn",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"from /f/pics"
]
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"b02113033af32feae9ff147dbbe3764039368d67d193885bd04e65c2e6beea9c.jpg"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 11
},
{
"sites": [
"reddit"
],
"task_id": 616,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pics",
"geolocation": null,
"intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
"instantiation_dict": {
"content": "Thanksgiving turkey",
"subreddit": "funny"
},
"intent": "Re-post the image of Thanksgiving turkey in this page to funny subreddit and note \"from /f/pics\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/funny",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"from /f/pics"
]
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"2e4fa0a328e653a97a7d07046291c298ef5b4e0d0c73a287f317ca86a8e8685f.jpg"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 11
},
{
"sites": [
"reddit"
],
"task_id": 617,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pics",
"geolocation": null,
"intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
"instantiation_dict": {
"content": "Firework",
"subreddit": "earthporn"
},
"intent": "Re-post the image of Firework in this page to earthporn subreddit and note \"from /f/pics\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/earthporn",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"from /f/pics"
]
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"92411be6af4e9ad5ccd3ccbaa01c10457bb00e704e99c58dd430de1a958307fd.jpg"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 11
},
{
"sites": [
"reddit"
],
"task_id": 618,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pics",
"geolocation": null,
"intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
"instantiation_dict": {
"content": "Wife's costume",
"subreddit": "funny"
},
"intent": "Re-post the image of Wife's costume in this page to funny subreddit and note \"from /f/pics\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/funny",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"from /f/pics"
]
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"6bfbf1280d28d84a9261695f0cac5a90addaaff1174807a2b381fdc159f3ed00.jpg"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 11
},
{
"sites": [
"reddit"
],
"task_id": 619,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pics",
"geolocation": null,
"intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
"instantiation_dict": {
"content": "costume contest",
"subreddit": "funny"
},
"intent": "Re-post the image of costume contest in this page to funny subreddit and note \"from /f/pics\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/funny",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"from /f/pics"
]
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"bd8bc5f4c846aac4df08626faa3a34a7d47c8f3bdd92bf615a54afd939f063a7.jpg"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 11
},
{
"sites": [
"reddit"
],
"task_id": 620,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
"instantiation_dict": {
"issue": "deal with long-distance relationships"
},
"intent": "Ask for advice about deal with long-distance relationships in a subreddit for relations",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"long",
"distance",
"relation"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 12
},
{
"sites": [
"reddit"
],
"task_id": 621,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
"instantiation_dict": {
"issue": "cheat"
},
"intent": "Ask for advice about cheat in a subreddit for relations",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"cheat"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 12
},
{
"sites": [
"reddit"
],
"task_id": 622,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
"instantiation_dict": {
"issue": "sexual harassment"
},
"intent": "Ask for advice about sexual harassment in a subreddit for relations",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"sexual",
"harassment"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 12
},
{
"sites": [
"reddit"
],
"task_id": 623,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
"instantiation_dict": {
"issue": "gift for birthday"
},
"intent": "Ask for advice about gift for birthday in a subreddit for relations",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"gift",
"birthday"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 12
},
{
"sites": [
"reddit"
],
"task_id": 624,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
"instantiation_dict": {
"issue": "break-up remedy"
},
"intent": "Ask for advice about break-up remedy in a subreddit for relations",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"break",
"remedy"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 12
},
{
"sites": [
"reddit"
],
"task_id": 625,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"instantiation_dict": {
"topic": "the effectiveness of online learning"
},
"intent": "Create a discussion post about \"the effectiveness of online learning\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"your opinion",
"the effectiveness of online learning"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 13
},
{
"sites": [
"reddit"
],
"task_id": 626,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"instantiation_dict": {
"topic": "Iphone 14"
},
"intent": "Create a discussion post about \"Iphone 14\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/iphone",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"your opinion",
"Iphone 14"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 13
},
{
"sites": [
"reddit"
],
"task_id": 627,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"instantiation_dict": {
"topic": "Harry Potter movie series"
},
"intent": "Create a discussion post about \"Harry Potter movie series\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/movies",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"your opinion",
"Harry Potter movie series"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 13
},
{
"sites": [
"reddit"
],
"task_id": 628,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"instantiation_dict": {
"topic": "long distance relationship"
},
"intent": "Create a discussion post about \"long distance relationship\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/relationship_advice",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"your opinion",
"long distance relationship"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 13
},
{
"sites": [
"reddit"
],
"task_id": 629,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"instantiation_dict": {
"topic": "Fun thing to do in Pittsburgh"
},
"intent": "Create a discussion post about \"Fun thing to do in Pittsburgh\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/pittsburgh",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"your opinion",
"Fun thing to do in Pittsburgh |OR| Fun things to do in Pittsburgh"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 13
},
{
"sites": [
"reddit"
],
"task_id": 630,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
"instantiation_dict": {
"category": "noise-cancelling headphones",
"price": "$200",
"subreddit": "r/headphones"
},
"intent": "Ask for product recommendations for noise-cancelling headphones within a budget of $200 in r/headphones",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/headphones",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"200",
"noise-cancelling",
"headphone"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 15
},
{
"sites": [
"reddit"
],
"task_id": 631,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
"instantiation_dict": {
"category": "running shoes",
"price": "$100",
"subreddit": "r/sports"
},
"intent": "Ask for product recommendations for running shoes within a budget of $100 in r/sports",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/sports",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"100",
"running",
"shoes"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 15
},
{
"sites": [
"reddit"
],
"task_id": 632,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
"instantiation_dict": {
"category": "running shoes",
"price": "$500",
"subreddit": "r/sports"
},
"intent": "Ask for product recommendations for running shoes within a budget of $500 in r/sports",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/sports",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"500",
"running",
"shoes"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 15
},
{
"sites": [
"reddit"
],
"task_id": 633,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
"instantiation_dict": {
"category": "running pants",
"price": "$500",
"subreddit": "r/sports"
},
"intent": "Ask for product recommendations for running pants within a budget of $500 in r/sports",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/sports",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"500",
"running",
"pants"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 15
},
{
"sites": [
"reddit"
],
"task_id": 634,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
"instantiation_dict": {
"category": "used iphone",
"price": "$1000",
"subreddit": "r/iphone"
},
"intent": "Ask for product recommendations for used iphone within a budget of $1000 in r/iphone",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/iphone",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"1000",
"used iphone"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 15
},
{
"sites": [
"reddit"
],
"task_id": 635,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
"instantiation_dict": {
"category": "noise-cancelling headphones",
"price": "$200"
},
"intent": "Post in the most appropriate subreddit and ask for recommendations for noise-cancelling headphones products within a budget of $200",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/headphones",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"200",
"noise-cancelling",
"headphone"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 6100
},
{
"sites": [
"reddit"
],
"task_id": 636,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
"instantiation_dict": {
"category": "DIY toolkit",
"price": "$100"
},
"intent": "Post in the most appropriate subreddit and ask for recommendations for DIY toolkit products within a budget of $100",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/DIY",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"100",
"DIY",
"toolkit"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 6100
},
{
"sites": [
"reddit"
],
"task_id": 637,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
"instantiation_dict": {
"category": "sony headphones",
"price": "$500"
},
"intent": "Post in the most appropriate subreddit and ask for recommendations for sony headphones products within a budget of $500",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/headphones",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"500",
"sony headphone"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 6100
},
{
"sites": [
"reddit"
],
"task_id": 638,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
"instantiation_dict": {
"category": "must-have product in my life",
"price": "$30"
},
"intent": "Post in the most appropriate subreddit and ask for recommendations for must-have product in my life products within a budget of $30",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/BuyItForLife",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"30",
"must-have",
"product",
"life"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 6100
},
{
"sites": [
"reddit"
],
"task_id": 639,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
"instantiation_dict": {
"category": "used iphone",
"price": "$1000"
},
"intent": "Post in the most appropriate subreddit and ask for recommendations for used iphone products within a budget of $1000",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/iphone",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"1000",
"used iphone"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 6100
},
{
"sites": [
"reddit"
],
"task_id": 640,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
"instantiation_dict": {
"interest": "book reading",
"date": "March 15th",
"subreddit": "r/books"
},
"intent": "Post a notice on a virtual meetup for book reading enthusiasts on March 15th in the r/books subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"book reading",
"Mar 15th |OR| March 15th",
"virtual",
"meetup"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 16
},
{
"sites": [
"reddit"
],
"task_id": 641,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
"instantiation_dict": {
"interest": "Harry Potter",
"date": "July 8th",
"subreddit": "books"
},
"intent": "Post a notice on a virtual meetup for Harry Potter enthusiasts on July 8th in the books subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Harry Potter",
"Jul 8th |OR| July 8th",
"virtual",
"meetup"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 16
},
{
"sites": [
"reddit"
],
"task_id": 642,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
"instantiation_dict": {
"interest": "Big little lies",
"date": "Sep 10th",
"subreddit": "books"
},
"intent": "Post a notice on a virtual meetup for Big little lies enthusiasts on Sep 10th in the books subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Big little lies",
"Sep 10th |OR| September 10th",
"virtual",
"meetup"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 16
},
{
"sites": [
"reddit"
],
"task_id": 643,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
"instantiation_dict": {
"interest": "racing cars",
"date": "Oct 21st",
"subreddit": "nyc"
},
"intent": "Post a notice on a virtual meetup for racing cars enthusiasts on Oct 21st in the nyc subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/nyc",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"racing cars",
"Oct 21st |OR| October 21st",
"virtual",
"meetup"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 16
},
{
"sites": [
"reddit"
],
"task_id": 644,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
"instantiation_dict": {
"interest": "Tears of Kingdom",
"date": "Dec 15th",
"subreddit": "gaming"
},
"intent": "Post a notice on a virtual meetup for Tears of Kingdom enthusiasts on Dec 15th in the gaming subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"Tears of Kingdom",
"Dec 15th |OR| December 15th",
"virtual",
"meetup"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 16
},
{
"sites": [
"reddit"
],
"task_id": 645,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in {{subreddit}} subreddit about what could machine learning help the corresponding field.",
"instantiation_dict": {
"subreddit": "books"
},
"intent": "Post in books subreddit about what could machine learning help the corresponding field.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"machine learning",
"help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 19
},
{
"sites": [
"reddit"
],
"task_id": 646,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in {{subreddit}} subreddit about what could midjourney help the corresponding field.",
"instantiation_dict": {
"subreddit": "DIY"
},
"intent": "Post in DIY subreddit about what could midjourney help the corresponding field.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/diy",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"midjourney",
"help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 19
},
{
"sites": [
"reddit"
],
"task_id": 647,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in {{subreddit}} forum about what could open-source LLMs help the corresponding field.",
"instantiation_dict": {
"subreddit": "technology"
},
"intent": "Post in technology forum about what could open-source LLMs help the corresponding field.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/technology",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"open-source LLMs",
"help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 19
},
{
"sites": [
"reddit"
],
"task_id": 648,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in {{subreddit}} forum about what could large language models help the corresponding field.",
"instantiation_dict": {
"subreddit": "dataisbeautiful"
},
"intent": "Post in dataisbeautiful forum about what could large language models help the corresponding field.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/dataisbeautiful",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"large language models",
"help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 19
},
{
"sites": [
"reddit"
],
"task_id": 649,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Post in {{subreddit}} subreddit about what could diffusion model help the corresponding field.",
"instantiation_dict": {
"subreddit": "history"
},
"intent": "Post in history subreddit about what could diffusion model help the corresponding field.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/history",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"diffusion model",
"help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 19
},
{
"sites": [
"reddit"
],
"task_id": 650,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
"geolocation": null,
"intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the post",
"content_description": "I am a big fan of the bookorg"
},
"intent": "Reply to the post with my comment \"I am a big fan of the bookorg\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.comment__body').outerText",
"required_contents": {
"exact_match": "I am a big fan of the bookorg"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 23
},
{
"sites": [
"reddit"
],
"task_id": 651,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/pittsburgh/45899/driving-in-pittsburgh-summed-up-by-one-traffic-sign",
"geolocation": null,
"intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the post",
"content_description": "Yeah, pittsburgh traffic, you know..."
},
"intent": "Reply to the post with my comment \"Yeah, pittsburgh traffic, you know...\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/pittsburgh/45899/driving-in-pittsburgh-summed-up-by-one-traffic-sign",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.comment__body').outerText",
"required_contents": {
"exact_match": "Yeah, pittsburgh traffic, you know..."
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 23
},
{
"sites": [
"reddit"
],
"task_id": 652,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/technology/134852/ai-experts-disown-musk-backed-campaign-citing-their-research",
"geolocation": null,
"intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the post",
"content_description": "???"
},
"intent": "Reply to the post with my comment \"???\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/technology/134852/ai-experts-disown-musk-backed-campaign-citing-their-research",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.comment__body').outerText",
"required_contents": {
"exact_match": "???"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 23
},
{
"sites": [
"shopping"
],
"task_id": 653,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
"instantiation_dict": {
"product": "phone screen protector",
"order_id": "000000180"
},
"intent": "Fill the \"contact us\" form in the site for a refund on the phone screen protector I bought, stating that it broke after just three days of use. Also, ensure to include the order number #000000180 and the product SKU. Don't submit yet, I will check.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"000000180",
"B087QJN9W1"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 153
},
{
"sites": [
"shopping"
],
"task_id": 654,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
"instantiation_dict": {
"product": "bluetooth speaker",
"order_id": "161"
},
"intent": "Fill the \"contact us\" form in the site for a refund on the bluetooth speaker I bought, stating that it broke after just three days of use. Also, ensure to include the order number #161 and the product SKU. Don't submit yet, I will check.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"161",
"B09P7BFL4H"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 153
},
{
"sites": [
"shopping"
],
"task_id": 655,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
"instantiation_dict": {
"product": "iphone case",
"order_id": "180"
},
"intent": "Fill the \"contact us\" form in the site for a refund on the iphone case I bought, stating that it broke after just three days of use. Also, ensure to include the order number #180 and the product SKU. Don't submit yet, I will check.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"180",
"B087QJN9W1"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 153
},
{
"sites": [
"shopping"
],
"task_id": 656,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
"instantiation_dict": {
"product": "remote controller",
"order_id": "180"
},
"intent": "Fill the \"contact us\" form in the site for a refund on the remote controller I bought, stating that it broke after just three days of use. Also, ensure to include the order number #180 and the product SKU. Don't submit yet, I will check.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"180",
"B0041MSF2S"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 153
},
{
"sites": [
"shopping"
],
"task_id": 657,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
"instantiation_dict": {
"product": "speaker",
"order_id": "148"
},
"intent": "Fill the \"contact us\" form in the site for a refund on the speaker I bought, stating that it broke after just three days of use. Also, ensure to include the order number #148 and the product SKU. Don't submit yet, I will check.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"refund",
"broke",
"three days of use",
"148",
"B003FVW3VA"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 153
},
{
"sites": [
"gitlab"
],
"task_id": 658,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
"instantiation_dict": {
"repo": "a11yproject",
"issue": "401 bad gateway",
"account": "Roshanjossey",
"due": "the end of 2030"
},
"intent": "Create an issue in a11yproject repo with title \"401 bad gateway\". Assign the issue to Roshanjossey. Set due date to be the end of 2030",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
"required_contents": {
"exact_match": "401 bad gateway"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
"required_contents": {
"must_include": [
"Dec 31, 2030"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.assignee').outerText",
"required_contents": {
"must_include": [
"Roshan Jossey"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 327
},
{
"sites": [
"gitlab"
],
"task_id": 659,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
"instantiation_dict": {
"repo": "empathy-prompts",
"issue": "Integrating LLMs for better prompts",
"account": "Roshanjossey",
"due": "the beginning of Q2 2033"
},
"intent": "Create an issue in empathy-prompts repo with title \"Integrating LLMs for better prompts\". Assign the issue to Roshanjossey. Set due date to be the beginning of Q2 2033",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
"required_contents": {
"exact_match": "Integrating LLMs for better prompts"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
"required_contents": {
"must_include": [
"Apr 1, 2033"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.assignee').outerText",
"required_contents": {
"must_include": [
"Roshan Jossey"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 327
},
{
"sites": [
"gitlab"
],
"task_id": 660,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
"instantiation_dict": {
"repo": "dotfiles",
"issue": "add support for oh-my-zsh",
"account": "Abishek",
"due": "July 18 2033"
},
"intent": "Create an issue in dotfiles repo with title \"add support for oh-my-zsh\". Assign the issue to Abishek. Set due date to be July 18 2033",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/dotfiles/-/tree/main",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
"required_contents": {
"exact_match": "add support for oh-my-zsh"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
"required_contents": {
"must_include": [
"Jul 18, 2033"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.assignee').outerText",
"required_contents": {
"must_include": [
"Abishek S"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 327
},
{
"sites": [
"gitlab"
],
"task_id": 661,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open an issue to {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "ChatGPT",
"issue": "report the issue of connection refused"
},
"intent": "Open an issue to report the issue of connection refused in ChatGPT.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/convexegg/chatgpt/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"connection refused"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 328
},
{
"sites": [
"gitlab"
],
"task_id": 662,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open an issue to {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "aem-hacker",
"issue": "report experiencing \"OSError: [Errno 98] Address already in use\" during executions"
},
"intent": "Open an issue to report experiencing \"OSError: [Errno 98] Address already in use\" during executions in aem-hacker.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/0ang3el/aem-hacker/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"OSError: [Errno 98] Address already in use"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 328
},
{
"sites": [
"gitlab"
],
"task_id": 663,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open an issue to {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "metaseq",
"issue": "ask their plan on supporting Llama and other llama family models"
},
"intent": "Open an issue to ask their plan on supporting Llama and other llama family models in metaseq.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"llama"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 328
},
{
"sites": [
"gitlab"
],
"task_id": 664,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open an issue to {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "awesome-python",
"issue": "ask their plans on adding Python 3.11 related resources"
},
"intent": "Open an issue to ask their plans on adding Python 3.11 related resources in awesome-python.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/vinta/awesome-python/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"Python 3.11"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 328
},
{
"sites": [
"gitlab"
],
"task_id": 665,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open an issue to {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "a11y-syntax-highlighting",
"issue": "request adding support for MT theme editor"
},
"intent": "Open an issue to request adding support for MT theme editor in a11y-syntax-highlighting.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"MT theme editor"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 328
},
{
"sites": [
"gitlab"
],
"task_id": 666,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Submit a request to merge {{source_branch}} branch into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "dialog-component",
"target_branch": "dialog",
"reviewer": "Carol"
},
"intent": "Submit a request to merge dialog-component branch into dialog branch, assign Carol as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "dialog"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "dialog-component"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Caroline Stewart"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 667,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "dialog-component",
"target_branch": "bump-doctocat",
"reviewer": "primer"
},
"intent": "Submit a merge request for dialog-component branch to be merged into bump-doctocat branch, assign primer as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "bump-doctocat"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "dialog-component"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Primer"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 668,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "a11yproject.com/redesign",
"target_branch": "master",
"reviewer": "Justin Armstrong"
},
"intent": "Submit a merge request for a11yproject.com/redesign branch to be merged into master branch, assign Justin Armstrong as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "redesign"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "main"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Justin Armstrong"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 669,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/solarized-prism-theme",
"geolocation": null,
"intent_template": "Open a new issue to discuss the implementation of {{feature}}",
"instantiation_dict": {
"feature": "dark mode"
},
"intent": "Open a new issue to discuss the implementation of dark mode",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/solarized-prism-theme/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"implementation",
"dark mode"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 337
},
{
"sites": [
"gitlab"
],
"task_id": 670,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/dotfiles",
"geolocation": null,
"intent_template": "Open a new issue to discuss the implementation of {{feature}}",
"instantiation_dict": {
"feature": "default plugins for .zsh"
},
"intent": "Open a new issue to discuss the implementation of default plugins for .zsh",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/dotfiles/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.detail-page-description').outerText",
"required_contents": {
"must_include": [
"implementation",
"default plugins",
"zsh"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 337
},
{
"sites": [
"shopping",
"reddit"
],
"task_id": 671,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
"instantiation_dict": {
"product": "Sony Computer Entertainment VR",
"rating": "2 stars and less"
},
"intent": "Gather the titles of Sony Computer Entertainment VR reviews with 2 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Sony Computer Entertainment VR\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__title').outerText",
"required_contents": {
"exact_match": "real user feedback on Sony Computer Entertainment VR"
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"didn't last a year without issues",
"Disappointing. Didn't last long before it stopped powering on and needed to be sent in for repair.",
"Received used items!!"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 101
},
{
"sites": [
"shopping",
"reddit"
],
"task_id": 672,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
"instantiation_dict": {
"product": "Nintendo Switch Fortnite Wildcat Console EU",
"rating": "3 stars and less"
},
"intent": "Gather the titles of Nintendo Switch Fortnite Wildcat Console EU reviews with 3 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Nintendo Switch Fortnite Wildcat Console EU\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__title').outerText",
"required_contents": {
"exact_match": "real user feedback on Nintendo Switch Fortnite Wildcat Console EU"
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"EU charger and wild cat card doesn\u2019t even work!",
"REFUND REJECTED",
"Charging port not compatible",
"not compatible in the US",
"Wildcard Bonus Credits Not Redeemable!",
"Code not available!!"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 101
},
{
"sites": [
"shopping",
"reddit"
],
"task_id": 673,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
"instantiation_dict": {
"product": "Racing Wheel Overdrive for Xbox X",
"rating": "1 star"
},
"intent": "Gather the titles of Racing Wheel Overdrive for Xbox X reviews with 1 star rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Racing Wheel Overdrive for Xbox X\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__title').outerText",
"required_contents": {
"exact_match": "real user feedback on Racing Wheel Overdrive for Xbox X"
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"Unable to set neutral steering",
"Doesn\u2019t work with PC",
"Crazy problems in automatic mode",
"pedals stopped working",
"Only works with certain games"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 101
},
{
"sites": [
"shopping",
"reddit"
],
"task_id": 674,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
"instantiation_dict": {
"product": "Doc and Pies Arcade Factory Cocktail Arcade Machine",
"rating": "3 stars and less"
},
"intent": "Gather the titles of Doc and Pies Arcade Factory Cocktail Arcade Machine reviews with 3 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Doc and Pies Arcade Factory Cocktail Arcade Machine\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__title').outerText",
"required_contents": {
"exact_match": "real user feedback on Doc and Pies Arcade Factory Cocktail Arcade Machine"
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"Poorly Made Exterior. Consider a different Company.",
"piece of junk ,..can't believe I spent money on this !!!!",
"Based arrived broken but game itself works"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 101
},
{
"sites": [
"shopping",
"reddit"
],
"task_id": 675,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
"instantiation_dict": {
"product": "HORI 3D Surround Gaming Neckset",
"rating": "2 stars and less"
},
"intent": "Gather the titles of HORI 3D Surround Gaming Neckset reviews with 2 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on HORI 3D Surround Gaming Neckset\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/gaming",
"program_html": [
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__title').outerText",
"required_contents": {
"exact_match": "real user feedback on HORI 3D Surround Gaming Neckset"
}
},
{
"url": "func:reddit_get_post_url('__last_url__')",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"Not worth it for PC users",
"I really wanted to like this.",
"I wish this was better..."
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 101
},
{
"sites": [
"shopping_admin"
],
"task_id": 676,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Lookup orders that are {{status}}",
"instantiation_dict": {
"status": "suspected of being fraudulent"
},
"intent": "Lookup orders that are suspected of being fraudulent",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales/order/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
"required_contents": {
"must_include": [
"Suspected Fraud"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 253
},
{
"sites": [
"shopping_admin"
],
"task_id": 677,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Lookup orders that are {{status}}",
"instantiation_dict": {
"status": "processing"
},
"intent": "Lookup orders that are processing",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales/order/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
"required_contents": {
"must_include": [
"Processing"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 253
},
{
"sites": [
"shopping_admin"
],
"task_id": 678,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Lookup orders that are {{status}}",
"instantiation_dict": {
"status": "canceled"
},
"intent": "Lookup orders that are canceled",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales/order/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
"required_contents": {
"must_include": [
"Canceled"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 253
},
{
"sites": [
"shopping_admin"
],
"task_id": 679,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Lookup orders that are {{status}}",
"instantiation_dict": {
"status": "completed"
},
"intent": "Lookup orders that are completed",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales/order/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
"required_contents": {
"must_include": [
"Complete"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 253
},
{
"sites": [
"shopping_admin"
],
"task_id": 680,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Lookup orders that are {{status}}",
"instantiation_dict": {
"status": "on hold"
},
"intent": "Lookup orders that are on hold",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales/order/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
"required_contents": {
"must_include": [
"On Hold"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 253
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 681,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
"instantiation_dict": {
"topic": "gan implementation"
},
"intent": "Find a GitLab repository related to gan implementation and make a Reddit post linking to it in a relevant subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning |OR| __REDDIT__/f/technology",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"gan implementation",
"__GITLAB__/eriklindernoren/PyTorch-GAN"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 116
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 682,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
"instantiation_dict": {
"topic": "chatGPT"
},
"intent": "Find a GitLab repository related to chatGPT and make a Reddit post linking to it in a relevant subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/singularity |OR| __REDDIT__/f/deeplearning |OR| __REDDIT__/f/technology",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"chatgpt",
"__GITLAB__/convexegg/chatgpt"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 116
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 683,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
"instantiation_dict": {
"topic": "metaseq"
},
"intent": "Find a GitLab repository related to metaseq and make a Reddit post linking to it in a relevant subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText",
"required_contents": {
"must_include": [
"metaseq",
"__GITLAB__/root/metaseq"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 116
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 684,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
"instantiation_dict": {
"repo": "byteblaze/cloud-to-butt",
"subreddit": "LifeProTips"
},
"intent": "Promote byteblaze/cloud-to-butt to subreddit LifeProTips with the description from the repo itself.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/LifeProTips",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"Chrome extension that replaces occurrences of 'the cloud' with 'my butt'",
"__GITLAB__/byteblaze/cloud-to-butt"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 117
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 685,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
"instantiation_dict": {
"repo": "byteblaze/dotfiles",
"subreddit": "aww"
},
"intent": "Promote byteblaze/dotfiles to subreddit aww with the description from the repo itself.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/aww",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"Computer setup",
"__GITLAB__/byteblaze/dotfiles"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 117
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 686,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
"instantiation_dict": {
"repo": "auth0/angular-storage",
"subreddit": "technology"
},
"intent": "Promote auth0/angular-storage to subreddit technology with the description from the repo itself.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/technology",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"A storage library for AngularJS done right",
"__GITLAB__/auth0/angular-storage"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 117
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 687,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
"instantiation_dict": {
"repo": "koush/AndroidAsync",
"subreddit": "funny"
},
"intent": "Promote koush/AndroidAsync to subreddit funny with the description from the repo itself.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/funny",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"Asynchronous socket, http(s) (client+server) and websocket library for android. Based on nio, not threads.",
"__GITLAB__/koush/AndroidAsync"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 117
},
{
"sites": [
"reddit",
"gitlab"
],
"task_id": 688,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
"instantiation_dict": {
"repo": "lahwaacz/arch-wiki-docs",
"subreddit": "science"
},
"intent": "Promote lahwaacz/arch-wiki-docs to subreddit science with the description from the repo itself.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__REDDIT__/f/science",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
"required_contents": {
"must_include": [
"A script to download pages from Arch Wiki for offline browsing",
"__GITLAB__/lahwaacz/arch-wiki-docs"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 117
},
{
"sites": [
"shopping"
],
"task_id": 689,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
"instantiation_dict": {
"reason": "I am a loyal customer"
},
"intent": "Draft an email to the shop owner via their contact us function for a coupon as I am a loyal customer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"coupon",
"I am a loyal customer"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 163
},
{
"sites": [
"shopping"
],
"task_id": 690,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
"instantiation_dict": {
"reason": "they promised me a coupon last time"
},
"intent": "Draft an email to the shop owner via their contact us function for a coupon as they promised me a coupon last time",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"coupon",
"promised"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 163
},
{
"sites": [
"shopping"
],
"task_id": 691,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
"instantiation_dict": {
"reason": "I plan to make a bulk purchase"
},
"intent": "Draft an email to the shop owner via their contact us function for a coupon as I plan to make a bulk purchase",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"coupon",
"bulk purchase"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 163
},
{
"sites": [
"shopping"
],
"task_id": 692,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
"instantiation_dict": {
"reason": "I am a student"
},
"intent": "Draft an email to the shop owner via their contact us function for a coupon as I am a student",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"coupon",
"student"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 163
},
{
"sites": [
"shopping"
],
"task_id": 693,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
"instantiation_dict": {
"reason": "my refund is suppoed to be replaced by a coupon"
},
"intent": "Draft an email to the shop owner via their contact us function for a coupon as my refund is suppoed to be replaced by a coupon",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/contact",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
"required_contents": {
"must_include": [
"coupon",
"refund"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 163
},
{
"sites": [
"shopping_admin"
],
"task_id": 694,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
"instantiation_dict": {
"product": "Energy-Bulk Women Shirt",
"stock": "50",
"size": "S",
"color": "blue",
"price": "60"
},
"intent": "Add a simple product named Energy-Bulk Women Shirt with 50 in stock, available in size S and color blue, priced at $60",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/catalog/product",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "60.00"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[name]\"').value",
"required_contents": {
"must_include": [
"Energy-Bulk Women Shirt"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "50"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
"required_contents": {
"must_include": [
"top"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[size]\"').value",
"required_contents": {
"exact_match": "167"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[color]\"').value",
"required_contents": {
"exact_match": "50"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
"required_contents": {
"must_include": [
"tops"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 256
},
{
"sites": [
"shopping_admin"
],
"task_id": 695,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
"instantiation_dict": {
"product": "Energy-Bulk Man Yoga Pant",
"stock": "50",
"size": "38",
"color": "yellow",
"price": "69.99"
},
"intent": "Add a simple product named Energy-Bulk Man Yoga Pant with 50 in stock, available in size 38 and color yellow, priced at $69.99",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/catalog/product",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "69.99"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[name]\"').value",
"required_contents": {
"must_include": [
"Energy-Bulk Man Yoga Pant"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "50"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
"required_contents": {
"must_include": [
"bottom"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[size]\"').value",
"required_contents": {
"exact_match": "179"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[color]\"').value",
"required_contents": {
"exact_match": "60"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
"required_contents": {
"must_include": [
"bottoms"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 256
},
{
"sites": [
"shopping_admin"
],
"task_id": 696,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
"instantiation_dict": {
"product": "FancyBoy Man Causal Jeans",
"stock": "42",
"size": "34",
"color": "Blue",
"price": "169.99"
},
"intent": "Add a simple product named FancyBoy Man Causal Jeans with 42 in stock, available in size 34 and color Blue, priced at $169.99",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/catalog/product",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"product[name]\"').value",
"required_contents": {
"must_include": [
"FancyBoy Man Causal Jeans"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "42"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "169.99"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
"required_contents": {
"must_include": [
"bottom"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[size]\"').value",
"required_contents": {
"exact_match": "177"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[color]\"').value",
"required_contents": {
"exact_match": "50"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
"required_contents": {
"must_include": [
"bottoms"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 256
},
{
"sites": [
"shopping_admin"
],
"task_id": 697,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
"instantiation_dict": {
"product": "Swaatch Smart Watch",
"stock": "42",
"size": "uni-size",
"color": "Blue",
"price": "769.99"
},
"intent": "Add a simple product named Swaatch Smart Watch with 42 in stock, available in size uni-size and color Blue, priced at $769.99",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/catalog/product",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"product[name]\"').value",
"required_contents": {
"must_include": [
"Swaatch Smart Watch"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "42"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "769.99"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
"required_contents": {
"must_include": [
"gear"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[color]\"').value",
"required_contents": {
"exact_match": "50"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
"required_contents": {
"must_include": [
"watches"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 256
},
{
"sites": [
"shopping_admin"
],
"task_id": 698,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
"instantiation_dict": {
"product": "Lelelumon Yoga Mat",
"stock": "42",
"size": "uni-size",
"color": "black",
"price": "769.99"
},
"intent": "Add a simple product named Lelelumon Yoga Mat with 42 in stock, available in size uni-size and color black, priced at $769.99",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/catalog/product",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"product[name]\"').value",
"required_contents": {
"must_include": [
"Lelelumon Yoga Mat"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "42"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "769.99"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
"required_contents": {
"must_include": [
"gear"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"product[color]\"').value",
"required_contents": {
"exact_match": "49"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
"required_contents": {
"must_include": [
"fitness equipment"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 256
},
{
"sites": [
"shopping_admin"
],
"task_id": 699,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
"instantiation_dict": {
"topic": "spring sale",
"rule": "a 20 percent discount site-wide"
},
"intent": "Draft a new marketing price rule for spring sale that offers a 20 percent discount site-wide for all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"[name='name'\").value",
"required_contents": {
"must_include": [
"spring sale"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
"required_contents": {
"exact_match": "0"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"simple_action\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "by_percent"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"discount_amount\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "20"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 258
},
{
"sites": [
"shopping_admin"
],
"task_id": 700,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
"instantiation_dict": {
"topic": "fall discount",
"rule": "$10 discount on checkout"
},
"intent": "Draft a new marketing price rule for fall discount that offers $10 discount on checkout for all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"[name='name'\").value",
"required_contents": {
"must_include": [
"fall discount"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
"required_contents": {
"exact_match": "0"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"simple_action\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "cart_fixed"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"discount_amount\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "10"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 258
},
{
"sites": [
"shopping_admin"
],
"task_id": 701,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
"instantiation_dict": {
"topic": "Mother's day sale",
"rule": "$15 discount on checkout"
},
"intent": "Draft a new marketing price rule for Mother's day sale that offers $15 discount on checkout for all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"[name='name'\").value",
"required_contents": {
"must_include": [
"Mother's day sale"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
"required_contents": {
"exact_match": "0"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"simple_action\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "cart_fixed"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"discount_amount\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "15"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 258
},
{
"sites": [
"shopping_admin"
],
"task_id": 702,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
"instantiation_dict": {
"topic": "Pride Month",
"rule": "45% off on all products"
},
"intent": "Draft a new marketing price rule for Pride Month that offers 45% off on all products for all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"[name='name'\").value",
"required_contents": {
"must_include": [
"Pride Month"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
"required_contents": {
"exact_match": "0"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"simple_action\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "by_percent"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"discount_amount\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "45"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 258
},
{
"sites": [
"shopping_admin"
],
"task_id": 703,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
"instantiation_dict": {
"topic": "Thanks giving sale",
"rule": "$40 discount on checkout"
},
"intent": "Draft a new marketing price rule for Thanks giving sale that offers $40 discount on checkout for all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"[name='name'\").value",
"required_contents": {
"must_include": [
"Thanks giving sale"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
"required_contents": {
"exact_match": "0"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"simple_action\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "cart_fixed"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"discount_amount\"').value",
"prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
"required_contents": {
"exact_match": "40"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 258
},
{
"sites": [
"shopping_admin"
],
"task_id": 704,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
"instantiation_dict": {
"report": "sales order report",
"time_span": "for last month"
},
"intent": "Today is 3/15/2023, generate a sales order report for last month",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "2/1/23"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "2/28/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 268
},
{
"sites": [
"shopping_admin"
],
"task_id": 705,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
"instantiation_dict": {
"report": "sales order report",
"time_span": "over the last 45 days"
},
"intent": "Today is 3/15/2023, generate a sales order report over the last 45 days",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "1/29/23"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "3/15/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 268
},
{
"sites": [
"shopping_admin"
],
"task_id": 706,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
"instantiation_dict": {
"report": "refund report",
"time_span": "for Q1"
},
"intent": "Today is 3/15/2023, generate a refund report for Q1",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/refunded",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "1/1/23"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "3/31/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 268
},
{
"sites": [
"shopping_admin"
],
"task_id": 707,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
"instantiation_dict": {
"report": "sales order report",
"time_span": "for last year"
},
"intent": "Today is 3/15/2023, generate a sales order report for last year",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "1/1/22"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "12/31/22"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 268
},
{
"sites": [
"shopping_admin"
],
"task_id": 708,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
"instantiation_dict": {
"report": "tax report",
"time_span": "for this year"
},
"intent": "Today is 3/15/2023, generate a tax report for this year",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/tax/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "1/1/23"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "12/31/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 268
},
{
"sites": [
"shopping_admin"
],
"task_id": 709,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Create an {{type}} report from {{start_date}} to {{end_date}}",
"instantiation_dict": {
"type": "orders",
"start_date": "beginning of May 2021",
"end_date": "end of March 2022"
},
"intent": "Create an orders report from beginning of May 2021 to end of March 2022",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "5/1/21"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "3/31/22"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 271
},
{
"sites": [
"shopping_admin"
],
"task_id": 710,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
"instantiation_dict": {
"type": "shipping",
"start_date": "08/05/2022",
"end_date": "03/01/2023"
},
"intent": "Create a shipping report from 08/05/2022 to 03/01/2023",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/shipping",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "8/5/22"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "3/1/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 271
},
{
"sites": [
"shopping_admin"
],
"task_id": 711,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
"instantiation_dict": {
"type": "product view",
"start_date": "07/05/2021",
"end_date": "05/31/2023"
},
"intent": "Create a product view report from 07/05/2021 to 05/31/2023",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_product/viewed/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "7/5/21"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "5/31/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 271
},
{
"sites": [
"shopping_admin"
],
"task_id": 712,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
"instantiation_dict": {
"type": "coupons",
"start_date": "05/01/2021",
"end_date": "05/15/2023"
},
"intent": "Create a coupons report from 05/01/2021 to 05/15/2023",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/coupons/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "5/1/21"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "5/15/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 271
},
{
"sites": [
"shopping_admin"
],
"task_id": 713,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
"instantiation_dict": {
"type": "best sellers",
"start_date": "05/01/2022",
"end_date": "05/31/2023"
},
"intent": "Create a best sellers report from 05/01/2022 to 05/31/2023",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/reports/report_sales/bestsellers/",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_from\"').value",
"required_contents": {
"exact_match": "5/1/22"
}
},
{
"url": "last",
"locator": "document.querySelector('[id=\"sales_report_to\"').value",
"required_contents": {
"exact_match": "5/31/23"
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 271
},
{
"sites": [
"reddit"
],
"task_id": 714,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
"instantiation_dict": {
"subreddit": "gadgets",
"k": "1"
},
"intent": "Thumbs down the top 1 post ever in gadgets.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/gadgets/19459/a-custom-gaming-pc-built-inside-a-vintage-1940s-motorola",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 24
},
{
"sites": [
"reddit"
],
"task_id": 715,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
"instantiation_dict": {
"subreddit": "history",
"k": "2"
},
"intent": "Thumbs down the top 2 post ever in history.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/history/84338/the-scientist-who-discovered-sperm-was-so-grossed-out-he",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/history/105990/4-500-year-old-sumerian-temple-dedicated-to-mighty-thunder",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 24
},
{
"sites": [
"reddit"
],
"task_id": 716,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
"instantiation_dict": {
"k": "3",
"subreddit": "books"
},
"intent": "Thumbs down the top 3 post ever in books.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/books/81371/the-letters-of-t-s-eliot-to-emily-hale-that-were-kept-sealed",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/books/59447/appalachian-prison-book-project-seeks-notebook-donations-the",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 24
},
{
"sites": [
"reddit"
],
"task_id": 717,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
"instantiation_dict": {
"k": "4",
"subreddit": "movies"
},
"intent": "Thumbs down the top 4 post ever in movies.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/movies/86174/who-will-win-the-oscar-for-actress-in-a-supporting-role",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/movies/86029/who-will-win-the-oscar-for-film-editing",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/movies/86055/cindy-williams-dies-laverne-amp-shirley-star-who-appeared-in",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/movies/42682/michelle-yeoh-to-receive-palm-springs-film-festival-s",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 24
},
{
"sites": [
"reddit"
],
"task_id": 718,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
"instantiation_dict": {
"k": "5",
"subreddit": "technology"
},
"intent": "Thumbs down the top 5 post ever in technology.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/technology/48670/brain-cancer-vaccine-succeeds-at-prolonging-survival-in",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/technology/134696/india-cuts-internet-for-27-million-people-amid-search-for",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/technology/48785/us-judge-orders-amazon-to-cease-and-desist-anti-union",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/technology/70354/activision-s-boston-studio-workers-announce-unionization",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/technology/70233/social-media-influencers-are-charged-with-feeding-followers",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 24
},
{
"sites": [
"reddit"
],
"task_id": 719,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"subreddit": "wallstreetbets",
"user": "ThetaGang_wsb"
},
"intent": "Like all submissions created by ThetaGang_wsb in subreddit wallstreetbets",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/wallstreetbets/29478/how-will-airbnb-close-following-their-earnings-report-on",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/wallstreetbets/29458/how-much-will-the-federal-reserve-raise-interest-rates-in",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 25
},
{
"sites": [
"reddit"
],
"task_id": 720,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"subreddit": "earthporn",
"user": "CameronKelsey"
},
"intent": "Like all submissions created by CameronKelsey in subreddit earthporn",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/EarthPorn/98332/my-favorite-place-on-the-planet-henry-s-fork-of-the-snake",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/EarthPorn/98297/2-years-later-this-is-still-one-of-the-most-incredible",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/EarthPorn/98256/i-can-t-wait-for-all-this-green-to-start-coming-back-little",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 25
},
{
"sites": [
"reddit"
],
"task_id": 721,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "UniversityofBath",
"subreddit": "IAmA"
},
"intent": "Like all submissions created by UniversityofBath in subreddit IAmA",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/IAmA/119742/hi-i-m-vienne-a-doctoral-student-at-the-university-of-bath-i",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/119719/hello-reddit-i-m-nazia-mehrban-a-lecturer-in-biotechnology",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/119714/i-m-ellie-jarvis-she-her-a-2nd-year-phd-student-in-the",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/55155/hi-i-m-dr-lucy-maddox-from-bath-university-uk-i-m-a-clinical",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/55142/we-re-sadeka-nujhat-hannah-leese-and-sandhya-moise-from-the",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/34032/we-re-sandhya-moise-david-phillips-and-chan-lee-from-the",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/13175/hi-i-m-kit-yates-i-m-a-mathematical-biologist-at-the",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/IAmA/13170/hello-i-m-dr-sara-fontani-from-the-university-of",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 25
},
{
"sites": [
"reddit"
],
"task_id": 722,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "Don_Gato1",
"subreddit": "new york"
},
"intent": "Like all submissions created by Don_Gato1 in subreddit new york",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/nyc/44650/fox-news-hosts-cast-new-york-as-crime-ridden-and-chaotic",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 25
},
{
"sites": [
"reddit"
],
"task_id": 723,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "FTorrez81",
"subreddit": "iphone13"
},
"intent": "Like all submissions created by FTorrez81 in subreddit iphone13",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"reference_answer_raw_annotation": "N/A",
"string_note": "FTorrez81 does not have any submissions in iphone13"
},
"intent_template_id": 25,
"string_note": "FTorrez81 has no submissions in subreddit iphone13"
},
{
"sites": [
"reddit"
],
"task_id": 724,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "Hrekires",
"subreddit": "news"
},
"intent": "Like all submissions created by Hrekires in subreddit news",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/news/129816/gov-whitmer-signs-bills-to-repeal-right-to-work-restore",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129808/disney-world-deal-with-union-will-raise-minimum-wage-to-18",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129794/judge-halts-wyoming-abortion-ban-days-after-it-took-effect",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129783/don-t-say-gay-lawmaker-pleads-guilty-to-covid-relief-fraud",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129594/arizona-gov-katie-hobbs-refuses-to-proceed-with-execution",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129508/tennessee-governor-oks-bill-to-cut-nashville-council-in-half",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43839/philadelphia-da-larry-krasner-impeached-by-pa-house",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43781/crypto-giant-ftx-to-file-for-bankruptcy-ceo-sam-bankman",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43572/sec-doj-investigating-crypto-platform-ftx",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43558/kansas-gov-laura-kelly-wins-re-election-defeating-gop",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 25
},
{
"sites": [
"reddit"
],
"task_id": 725,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"subreddit": "massachusetts",
"user": "RickyDontLoseThat"
},
"intent": "DisLike all submissions created by RickyDontLoseThat in subreddit massachusetts",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/massachusetts/84954/the-last-of-lincoln",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 726,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"subreddit": "earthporn",
"user": "jacyanthis"
},
"intent": "DisLike all submissions created by jacyanthis in subreddit earthporn",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "jacyanthis does not have any submissions in earthporn",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 727,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "PatientBuilder499",
"subreddit": "videos"
},
"intent": "DisLike all submissions created by PatientBuilder499 in subreddit videos",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/videos/115139/hundreds-of-civilian-turkish-volunteers-waiting-to-be-sent",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 728,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "sirbarani",
"subreddit": "sports"
},
"intent": "DisLike all submissions created by sirbarani in subreddit sports",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/sports/48303/iran-football-legend-daei-will-not-attend-world-cup-amid",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 729,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "AdamCannon",
"subreddit": "UpliftingNews"
},
"intent": "DisLike all submissions created by AdamCannon in subreddit UpliftingNews",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/UpliftingNews/16087/same-sex-marriage-is-now-legal-in-all-of-mexico-s-states",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 730,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
"instantiation_dict": {
"user": "Hrekires",
"subreddit": "news"
},
"intent": "DisLike all submissions created by Hrekires in subreddit news",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/news/129816/gov-whitmer-signs-bills-to-repeal-right-to-work-restore",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129808/disney-world-deal-with-union-will-raise-minimum-wage-to-18",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129794/judge-halts-wyoming-abortion-ban-days-after-it-took-effect",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129783/don-t-say-gay-lawmaker-pleads-guilty-to-covid-relief-fraud",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129594/arizona-gov-katie-hobbs-refuses-to-proceed-with-execution",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/129508/tennessee-governor-oks-bill-to-cut-nashville-council-in-half",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43839/philadelphia-da-larry-krasner-impeached-by-pa-house",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43781/crypto-giant-ftx-to-file-for-bankruptcy-ceo-sam-bankman",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43572/sec-doj-investigating-crypto-platform-ftx",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
},
{
"url": "__REDDIT__/f/news/43558/kansas-gov-laura-kelly-wins-re-election-defeating-gop",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-downvoted"
]
}
}
]
},
"intent_template_id": 1510
},
{
"sites": [
"reddit"
],
"task_id": 731,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
"instantiation_dict": {
"post": "Nvidia RTX 4090",
"content": "EDIT: This news aged well"
},
"intent": "Edit my post on Nvidia RTX 4090 by adding a line to the body that says \"EDIT: This news aged well\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/MachineLearning/1/nvidia-rtx-4090",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"must_include": [
"EDIT: This news aged well",
"Crazy device for ML!"
]
}
}
]
},
"intent_template_id": 27
},
{
"sites": [
"reddit"
],
"task_id": 732,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
"instantiation_dict": {
"post": "The Night Agent",
"content": "Done watching, pretty cool!"
},
"intent": "Edit my post on The Night Agent by adding a line to the body that says \"Done watching, pretty cool!\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/television/134868/the-night-agent-renewed-for-season-2-at-netflix",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"exact_match": "Done watching, pretty cool!"
}
}
]
},
"intent_template_id": 27
},
{
"sites": [
"reddit"
],
"task_id": 733,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
"instantiation_dict": {
"post": "Star Trek Starfleet Academy series",
"content": "Every watch makes me feel like a kid again"
},
"intent": "Edit my post on Star Trek Starfleet Academy series by adding a line to the body that says \"Every watch makes me feel like a kid again\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/television/135201/star-trek-starfleet-academy-series-from-alex-kurtzman-and",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"exact_match": "Every watch makes me feel like a kid again"
}
}
]
},
"intent_template_id": 27
},
{
"sites": [
"reddit"
],
"task_id": 734,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
"instantiation_dict": {
"post": "Ted Lasso",
"content": "Done watching. I love the renew!"
},
"intent": "Edit my post on Ted Lasso by adding a line to the body that says \"Done watching. I love the renew!\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/television/135156/ted-lasso-season-3-premiere-scores-870k-u-s-households-up-59",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"exact_match": "Done watching. I love the renew!"
}
}
]
},
"intent_template_id": 27
},
{
"sites": [
"reddit"
],
"task_id": 735,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
"instantiation_dict": {
"post": "Lord of the Rings",
"content": "The cast is amazing!"
},
"intent": "Edit my post on Lord of the Rings by adding a line to the body that says \"The cast is amazing!\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/television/135152/lord-of-the-rings-the-rings-of-power-season-2-cast-adds",
"locator": "document.querySelector('.submission__body').outerText",
"required_contents": {
"exact_match": "The cast is amazing!"
}
}
]
},
"intent_template_id": 27
},
{
"sites": [
"gitlab"
],
"task_id": 736,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to MIT license.",
"instantiation_dict": {
"repo": "gimmiethat.space and dotfiles"
},
"intent": "Make the LICENSE of gimmiethat.space and dotfiles to MIT license.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"MIT license",
"The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
]
}
},
{
"url": "__GITLAB__/byteblaze/dotfiles/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"MIT license",
"The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 737,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
"instantiation_dict": {
"location": "Carnegie Mellon University",
"sport_team": "Philadelphia 76ers",
"time": ""
},
"intent": "Show me the way from Carnegie Mellon University to the home stadium of Philadelphia 76ers ",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Wells Fargo Center",
"South Philadelphia Sports Complex"
]
}
},
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 94
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 738,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
"instantiation_dict": {
"location": "Carnegie Mellon University",
"sport_team": "Philadelphia 76ers",
"time": "in the 70th"
},
"intent": "Show me the way from Carnegie Mellon University to the home stadium of Philadelphia 76ers in the 70th",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"3601 South Broad Street",
"South Philadelphia"
]
}
},
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 94
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 739,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
"instantiation_dict": {
"location": "Carnegie Mellon University",
"sport_team": "Yankees",
"time": "in the 80th"
},
"intent": "Show me the way from Carnegie Mellon University to the home stadium of Yankees in the 80th",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Yankee Stadium",
"East 161st Street"
]
}
},
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 94
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 740,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
"instantiation_dict": {
"location": "Carnegie Mellon University",
"sport_team": "NYC NBA team",
"time": ""
},
"intent": "Show me the way from Carnegie Mellon University to the home stadium of NYC NBA team ",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Madison Square Garden",
"Pennsylvania Plaza",
"Manhattan",
"New York"
]
}
},
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 94
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 741,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
"instantiation_dict": {
"location": "Carnegie Mellon University",
"sport_team": "Boston home NBA team",
"time": ""
},
"intent": "Show me the way from Carnegie Mellon University to the home stadium of Boston home NBA team ",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"TD Garden",
"Boston",
"Massachusetts"
]
}
},
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 94
},
{
"sites": [
"gitlab"
],
"task_id": 742,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a new {{scope}} project \"planner\" and add {{account_list}} as members",
"instantiation_dict": {
"scope": "private",
"account_list": "Abishek, Vinta"
},
"intent": "Create a new private project \"planner\" and add Abishek, Vinta as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/planner",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"private"
]
}
},
{
"url": "__GITLAB__/byteblaze/planner/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@abisubramanya27",
"@vinta"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 743,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a new {{scope}} project \"web_arena\" and add {{account_list}} as members",
"instantiation_dict": {
"scope": "public",
"account_list": "Abishek, Vinta"
},
"intent": "Create a new public project \"web_arena\" and add Abishek, Vinta as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_arena",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"public"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_arena/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@abisubramanya27",
"@vinta"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 744,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a new {{scope}} project \"AutoAGI\" and add {{account_list}} as members",
"instantiation_dict": {
"scope": "public",
"account_list": "primer"
},
"intent": "Create a new public project \"AutoAGI\" and add primer as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/AutoAGI",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"public"
]
}
},
{
"url": "__GITLAB__/byteblaze/AutoAGI/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@primer"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 745,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a new {{scope}} project \"awesome-llms\" and add {{account_list}} as members",
"instantiation_dict": {
"scope": "public",
"account_list": "primer, convexegg, abishek"
},
"intent": "Create a new public project \"awesome-llms\" and add primer, convexegg, abishek as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome-llms",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"public"
]
}
},
{
"url": "__GITLAB__/byteblaze/awesome-llms/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@primer",
"@convexegg",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 746,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a new {{scope}} project \"llm_bulk_inference\" and add {{account_list}} as members",
"instantiation_dict": {
"scope": "private",
"account_list": "primer, convexegg, abishek"
},
"intent": "Create a new private project \"llm_bulk_inference\" and add primer, convexegg, abishek as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/llm_bulk_inference",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/llm_bulk_inference/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@primer",
"@convexegg",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 747,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
"instantiation_dict": {
"project_name": "awesome_web_agents",
"template": "blank",
"account_list": "Abishek, Vinta"
},
"intent": "Start a private project awesome_web_agents with blank template and add Abishek, Vinta as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_web_agents",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/awesome_web_agents/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initial commit"
]
}
},
{
"url": "__GITLAB__/byteblaze/awesome_web_agents/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@abisubramanya27",
"@vinta"
]
}
}
]
},
"intent_template_id": 2100
},
{
"sites": [
"gitlab"
],
"task_id": 748,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
"instantiation_dict": {
"project_name": "web_agent_android_xl",
"template": "Android",
"account_list": "primer, convexegg, abishek"
},
"intent": "Start a private project web_agent_android_xl with Android template and add primer, convexegg, abishek as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_agent_android_xl",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent_android_xl/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initialized from 'Android' project template"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent_android_xl/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@primer",
"@convexegg",
"@abisubramanya27"
]
}
}
]
},
"intent_template_id": 2100
},
{
"sites": [
"gitlab"
],
"task_id": 749,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
"instantiation_dict": {
"project_name": "project_site",
"template": "NodeJS",
"account_list": "primer, convexegg, vinta"
},
"intent": "Start a private project project_site with NodeJS template and add primer, convexegg, vinta as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/project_site",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/project_site/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initialized from 'NodeJS Express' project template"
]
}
},
{
"url": "__GITLAB__/byteblaze/project_site/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@primer",
"@convexegg",
"@vinta"
]
}
}
]
},
"intent_template_id": 2100
},
{
"sites": [
"gitlab"
],
"task_id": 750,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
"instantiation_dict": {
"project_name": "agi_index",
"template": "HTML",
"account_list": "Vinta Chen"
},
"intent": "Start a private project agi_index with HTML template and add Vinta Chen as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/agi_index",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/agi_index",
"locator": "document.querySelector('.home-panel-description-markdown').outerText",
"required_contents": {
"must_include": [
"Example plain HTML site using GitLab Pages: https://pages.gitlab.io/plain-html |OR| A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
]
}
},
{
"url": "__GITLAB__/byteblaze/agi_index/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"Vinta Chen"
]
}
}
]
},
"intent_template_id": 2100
},
{
"sites": [
"gitlab"
],
"task_id": 751,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
"instantiation_dict": {
"project_name": "AGISite",
"template": "JEKYLL",
"account_list": "Rohan and Vinta"
},
"intent": "Start a private project AGISite with JEKYLL template and add Rohan and Vinta as members",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/AGISite",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/AGISite",
"locator": "document.querySelector('.home-panel-description-markdown').outerText",
"required_contents": {
"must_include": [
"Example Jekyll site using GitLab Pages: https://pages.gitlab.io/jekyll |OR| A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
]
}
},
{
"url": "__GITLAB__/byteblaze/AGISite/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"@Seirdy",
"@vinta"
]
}
}
]
},
"intent_template_id": 2100
},
{
"sites": [
"gitlab"
],
"task_id": 752,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
"instantiation_dict": {
"project_name": "web_agent",
"template": "blank"
},
"intent": "Create a private blank repository called \"web_agent\" using the right template to speed up development.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_agent",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initial commit"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 753,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
"instantiation_dict": {
"project_name": "web_agent_android_xs",
"template": "Android"
},
"intent": "Create a private Android repository called \"web_agent_android_xs\" using the right template to speed up development.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_agent_android_xs",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent_android_xs/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initialized from 'Android' project template"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 754,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
"instantiation_dict": {
"project_name": "web_agent_nodejs",
"template": "NodeJS"
},
"intent": "Create a private NodeJS repository called \"web_agent_nodejs\" using the right template to speed up development.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_agent_nodejs",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent_nodejs/-/commits",
"locator": "",
"required_contents": {
"must_include": [
"Initialized from 'NodeJS Express' project template"
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 755,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
"instantiation_dict": {
"project_name": "web_agent_index",
"template": "HTML"
},
"intent": "Create a private HTML repository called \"web_agent_index\" using the right template to speed up development.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/web_agent_index",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/web_agent_index",
"locator": "document.querySelector('.home-panel-description-markdown').outerText",
"required_contents": {
"must_include": [
"Example plain HTML site using GitLab Pages: https://pages.gitlab.io/plain-html |OR| A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"gitlab"
],
"task_id": 756,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
"instantiation_dict": {
"project_name": "11711_gitlab",
"template": "JEKYLL"
},
"intent": "Create a private JEKYLL repository called \"11711_gitlab\" using the right template to speed up development.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/11711_gitlab",
"locator": "document.querySelector('.visibility-icon').getAttribute('title')",
"required_contents": {
"must_include": [
"Private"
]
}
},
{
"url": "__GITLAB__/byteblaze/11711_gitlab",
"locator": "document.querySelector('.home-panel-description-markdown').outerText",
"required_contents": {
"must_include": [
"Example Jekyll site using GitLab Pages: https://pages.gitlab.io/jekyll |OR| A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
]
}
}
]
},
"intent_template_id": 332
},
{
"sites": [
"map"
],
"task_id": 757,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the path and travel time from {{city1}} to {{city2}}.",
"instantiation_dict": {
"city1": "home of the 1980 Super Bowl champions",
"city2": "home of the 1991 Super Bowl champions"
},
"intent": "Show me the path and travel time from home of the 1980 Super Bowl champions to home of the 1991 Super Bowl champions.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"New York"
]
}
}
]
},
"intent_template_id": 42
},
{
"sites": [
"map"
],
"task_id": 758,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the path and travel time from {{city1}} to {{city2}}.",
"instantiation_dict": {
"city1": "the big apple",
"city2": "biggest city in Maine"
},
"intent": "Show me the path and travel time from the big apple to biggest city in Maine.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"New York"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Portland",
"Maine"
]
}
}
]
},
"intent_template_id": 42
},
{
"sites": [
"map",
"shopping_admin"
],
"task_id": 759,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the route and driving time from {{city1}} to {{city2}}",
"instantiation_dict": {
"city1": "the city where my E-commerce customer Sophia Young lives",
"city2": "New York City"
},
"intent": "Show me the route and driving time from the city where my E-commerce customer Sophia Young lives to New York City",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Boston"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"New York"
]
}
}
]
},
"intent_template_id": 42
},
{
"sites": [
"map",
"shopping_admin"
],
"task_id": 760,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the route and driving time from {{city1}} to {{city2}}",
"instantiation_dict": {
"city1": "Allentown, PA",
"city2": "the city where my E-commerce customer Amanda Kim lives"
},
"intent": "Show me the route and driving time from Allentown, PA to the city where my E-commerce customer Amanda Kim lives",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Allentown"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Hoboken",
"New Jersey"
]
}
}
]
},
"intent_template_id": 42
},
{
"sites": [
"map"
],
"task_id": 761,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Get directions from {{location/address_1}} to {{location/address_2}} using {{transportation}} options.",
"instantiation_dict": {
"location/address_1": "Carnegie Science Museum",
"location/address_2": "Hunt library CMU",
"transportation": "walk"
},
"intent": "Get directions from Carnegie Science Museum to Hunt library CMU using walk options.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Science Center",
"Allegheny County",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Hunt Library",
"Pittsburgh"
]
}
}
]
},
"intent_template_id": 54
},
{
"sites": [
"map"
],
"task_id": 762,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Get directions from {{location/address_1}} to {{location/address_2}} using {{transportation}} options.",
"instantiation_dict": {
"location/address_1": "Carnegie Music Hall in NYC",
"location/address_2": "Carnegie Mellon University",
"transportation": "driving"
},
"intent": "Get directions from Carnegie Music Hall in NYC to Carnegie Mellon University using driving options.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Carnegie Hall",
"West 57th Street",
"Manhattan",
"New York"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Carnegie Mellon University",
"Pittsburgh"
]
}
}
]
},
"intent_template_id": 54
},
{
"sites": [
"map"
],
"task_id": 763,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
"instantiation_dict": {
"store": "Trader Joe's",
"location": "401 Shady Ave, Pittsburgh"
},
"intent": "Find the walkway to the closest Trader Joe's from 401 Shady Ave, Pittsburgh.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"401, Shady Avenue, Shadyside"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Trader Joe's, 6343, Penn Avenue, East Liberty"
]
}
}
]
},
"intent_template_id": 75
},
{
"sites": [
"map"
],
"task_id": 764,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
"instantiation_dict": {
"store": "Target",
"location": "401 Shady Ave, Pittsburgh"
},
"intent": "Find the walkway to the closest Target from 401 Shady Ave, Pittsburgh.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"401, Shady Avenue, Shadyside"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Target, 6231, Penn Avenue, East Liberty"
]
}
}
]
},
"intent_template_id": 75
},
{
"sites": [
"map"
],
"task_id": 765,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
"instantiation_dict": {
"store": "Japanese food market",
"location": "401 Shady Ave, Pittsburgh"
},
"intent": "Find the walkway to the closest Japanese food market from 401 Shady Ave, Pittsburgh.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"401, Shady Avenue, Shadyside"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Tokyo Japanese Food Store, 5855, Ellsworth Avenue, Shadyside"
]
}
}
]
},
"intent_template_id": 75
},
{
"sites": [
"map"
],
"task_id": 766,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
"instantiation_dict": {
"store": "grocessory owned by Amazon",
"location": "401 Shady Ave, Pittsburgh"
},
"intent": "Find the walkway to the closest grocessory owned by Amazon from 401 Shady Ave, Pittsburgh.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"401, Shady Avenue, Shadyside"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Whole Foods Market, 5700, Penn Avenue, East Liberty"
]
}
}
]
},
"intent_template_id": 75
},
{
"sites": [
"map"
],
"task_id": 767,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
"instantiation_dict": {
"store": "chain grocessory owned by a local business",
"location": "401 Shady Ave, Pittsburgh"
},
"intent": "Find the walkway to the closest chain grocessory owned by a local business from 401 Shady Ave, Pittsburgh.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "2"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"401, Shady Avenue, Shadyside"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Giant Eagle, 5550, Centre Avenue, Shadyside"
]
}
}
]
},
"intent_template_id": 75
},
{
"sites": [
"shopping_admin"
],
"task_id": 768,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{quantity}} {{product}} arrived, update the stock",
"instantiation_dict": {
"quantity": "5",
"product": "blue Cronus yoga pants with size 33"
},
"intent": "5 blue Cronus yoga pants with size 33 arrived, update the stock",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/872/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "5"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/872/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 241
},
{
"sites": [
"shopping_admin"
],
"task_id": 769,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "We've received {{quantity}} {{product}}, please update the inventory.",
"instantiation_dict": {
"quantity": "378",
"product": "brown Aero daily fitness tee in every size"
},
"intent": "We've received 378 brown Aero daily fitness tee in every size, please update the inventory.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/544/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "478"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/547/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "478"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/550/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "478"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/553/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "478"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/556/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "478"
}
}
]
},
"intent_template_id": 241
},
{
"sites": [
"shopping_admin"
],
"task_id": 770,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "We've received {{quantity}}, update the inventory.",
"instantiation_dict": {
"quantity": "12 white Cora parachute pant of size 28 and 56 blue of size 29"
},
"intent": "We've received 12 white Cora parachute pant of size 28 and 56 blue of size 29, update the inventory.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1836/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "112"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1838/",
"locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
"required_contents": {
"exact_match": "156"
}
}
]
},
"intent_template_id": 241
},
{
"sites": [
"shopping_admin"
],
"task_id": 771,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Approve the positive reviews to display in our store.",
"instantiation_dict": {},
"intent": "Approve the positive reviews to display in our store.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/352",
"locator": "document.querySelector('[name=\"status_id\"').value",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/349",
"locator": "document.querySelector('[name=\"status_id\"').value",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/347",
"locator": "document.querySelector('[name=\"status_id\"').value",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 243
},
{
"sites": [
"shopping_admin"
],
"task_id": 772,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "pending negative reviews for Circe fleece"
},
"intent": "Delete all pending negative reviews for Circe fleece",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/999",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
}
]
},
"intent_template_id": 246
},
{
"sites": [
"shopping_admin"
],
"task_id": 773,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "pending negative reviews"
},
"intent": "Delete all pending negative reviews",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/351",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/353",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
}
]
},
"intent_template_id": 246
},
{
"sites": [
"shopping_admin"
],
"task_id": 774,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "pending reviews with less than 4 stars"
},
"intent": "Delete all pending reviews with less than 4 stars",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/351",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/353",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/349",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
}
]
},
"intent_template_id": 246
},
{
"sites": [
"shopping_admin"
],
"task_id": 775,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "reviews from the scammer Arden"
},
"intent": "Delete all reviews from the scammer Arden",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/51",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
}
]
},
"intent_template_id": 246
},
{
"sites": [
"shopping_admin"
],
"task_id": 776,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "reviews from the scammer Carlo"
},
"intent": "Delete all reviews from the scammer Carlo",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/93",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
},
{
"url": "__SHOPPING_ADMIN__/review/product/edit/id/109",
"locator": "",
"required_contents": {
"must_include": [
"Rating isn't Available"
]
}
}
]
},
"intent_template_id": 246
},
{
"sites": [
"shopping_admin"
],
"task_id": 777,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "$5",
"action": "Reduce",
"config": "green Hollister backyard sweater in all size"
},
"intent": "Reduce the price of green Hollister backyard sweater in all size by $5",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/120/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "47.00"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/117/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "47.00"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/114/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "47.00"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/111/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "47.00"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/123/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "47.00"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"shopping_admin"
],
"task_id": 778,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "13.5%",
"action": "Reduce",
"config": "size 28 Sahara leggings"
},
"intent": "Reduce the price of size 28 Sahara leggings by 13.5%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1841/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "64.88"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1842/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "64.88"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1843/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "64.88"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"shopping_admin"
],
"task_id": 779,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "15%",
"action": "Reduce",
"config": "yellow shirts from Gwyn Endurance in all size below L"
},
"intent": "Reduce the price of yellow shirts from Gwyn Endurance in all size below L by 15%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1559/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "20.40"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1562/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "20.40"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1565/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "20.40"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"shopping_admin"
],
"task_id": 780,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "$17",
"action": "Increase",
"config": "white Ingrid Running with size L and above"
},
"intent": "Increase the price of white Ingrid Running with size L and above by $17",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1264/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "64.00"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1267/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "64.00"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"shopping_admin"
],
"task_id": 781,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "37%",
"action": "Increase",
"config": "black fitness tshirts from Desiree with size XS"
},
"intent": "Increase the price of black fitness tshirts from Desiree with size XS by 37%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1573/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "32.88"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"shopping_admin"
],
"task_id": 782,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "{{action}} the price of {{config}} by {{amount}}",
"instantiation_dict": {
"amount": "23%",
"action": "Increase",
"config": "all blue running tshirts in extra small and small sizes"
},
"intent": "Increase the price of all blue running tshirts in extra small and small sizes by 23%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/496/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "22.33"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/499/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "22.33"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/479/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "21.56"
}
},
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/482/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "21.56"
}
}
]
},
"intent_template_id": 742
},
{
"sites": [
"gitlab"
],
"task_id": 783,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
"instantiation_dict": {
"repo": "kkroening/ffmpeg-python",
"role": "maintainer",
"user_list": [
"yjlou",
"a11yproject"
]
},
"intent": "Add the following users to repo kkroening/ffmpeg-python as maintainer: ['yjlou', 'a11yproject']",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "not the owner of the repo so cannot add maintainer",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 351
},
{
"sites": [
"gitlab"
],
"task_id": 784,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/CellularPrivacy/Android-IMSI-Catcher-Detector",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
"instantiation_dict": {
"branch_name": "main",
"attribute": "email address"
},
"intent": "Tell me the email address of the contributor who has the most commits to branch main",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"secupwn@users.noreply.github.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "secupwn@users.noreply.github.com"
},
"intent_template_id": 316
},
{
"sites": [
"gitlab"
],
"task_id": 785,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/CellularPrivacy/Android-IMSI-Catcher-Detector",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
"instantiation_dict": {
"branch_name": "gh-page",
"attribute": "email address"
},
"intent": "Tell me the email address of the contributor who has the most commits to branch gh-page",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"secupwn@users.noreply.github.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "secupwn@users.noreply.github.com"
},
"intent_template_id": 316
},
{
"sites": [
"gitlab"
],
"task_id": 786,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/vinta/awesome-python",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
"instantiation_dict": {
"branch_name": "main",
"attribute": "number of commits"
},
"intent": "Tell me the number of commits of the contributor who has the most commits to branch main",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"412"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "412"
},
"intent_template_id": 316
},
{
"sites": [
"gitlab"
],
"task_id": 787,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/dehenne/awesome-visibility/-/graphs/master",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
"instantiation_dict": {
"branch_name": "main",
"attribute": "number of followers"
},
"intent": "Tell me the number of followers of the contributor who has the most commits to branch main",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 316
},
{
"sites": [
"gitlab"
],
"task_id": 788,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/amwhalen/archive-my-tweets/-/tree/github/fork/chtitux/addRssFeed",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
"instantiation_dict": {
"branch_name": "php52",
"attribute": "full name, gitlab account name, location and email address"
},
"intent": "Tell me the full name, gitlab account name, location and email address of the contributor who has the most commits to branch php52",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Andrew M. Whalen",
"amwhalen",
"Massachusetts",
"github@amwhalen.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Andrew M. Whalen, amwhalen, Massachusetts, github@amwhalen.com"
},
"intent_template_id": 316
},
{
"sites": [
"gitlab"
],
"task_id": 789,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue {{issue}} in {{repo}}.",
"instantiation_dict": {
"repo": "huggingface dataset",
"issue": "asking about do they have any plan on supporting Webagent in the next quater"
},
"intent": "Create an issue asking about do they have any plan on supporting Webagent in the next quater in huggingface dataset.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "huggingface repo does not exist",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 328
},
{
"sites": [
"shopping_admin"
],
"task_id": 790,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Delete all {{review_type}}",
"instantiation_dict": {
"review_type": "negative reviews for Sybil running short"
},
"intent": "Delete all negative reviews for Sybil running short",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "No negative reviews for sybil running short",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 246
},
{
"sites": [
"gitlab",
"reddit"
],
"task_id": 791,
"require_login": true,
"storage_state": "./.auth/gitlab.reddit_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make a folder named {{directory}} on the {{gitlab_repo}} repo and include a file called urls.txt that consists of the links to the 5 most recent posts from {{subreddit}}.",
"instantiation_dict": {
"directory": "car",
"subreddit": "cars",
"gitlab_repo": "gimmiethat.space"
},
"intent": "Make a folder named car on the gimmiethat.space repo and include a file called urls.txt that consists of the links to the 5 most recent posts from cars.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "the subreddit does not exist",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 84
},
{
"sites": [
"shopping"
],
"task_id": 792,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
"instantiation_dict": {
"product_category": "Beauty & Personal Care",
"dollar_value": "under 20"
},
"intent": "Buy the highest rated product from the Beauty & Personal Care category within a budget under 20.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The website does not support sorting by rating and there are too many products in this category",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 172
},
{
"sites": [
"shopping"
],
"task_id": 793,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
"instantiation_dict": {
"product_category": "Men clothing",
"dollar_value": "above 50 but under 129.99"
},
"intent": "Buy the highest rated product from the Men clothing category within a budget above 50 but under 129.99.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The website does not support sorting by rating and there are too many products in this category",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 172
},
{
"sites": [
"shopping"
],
"task_id": 794,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Change the delivery address for my most recent order to {{address}}.",
"instantiation_dict": {
"address": "4000 Forbes Ave, Pittsburgh, PA"
},
"intent": "Change the delivery address for my most recent order to 4000 Forbes Ave, Pittsburgh, PA.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The shipping address cannot be changed after an order is placed in the website.",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 191
},
{
"sites": [
"shopping"
],
"task_id": 795,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Change the delivery address for my most recent order to {{address}}.",
"instantiation_dict": {
"address": "6726 McPherson Blvd, Pittsburgh, PA"
},
"intent": "Change the delivery address for my most recent order to 6726 McPherson Blvd, Pittsburgh, PA.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The shipping address cannot be changed after an order is placed in the website.",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 191
},
{
"sites": [
"shopping"
],
"task_id": 796,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Change the delivery address for my most recent order to {{address}}.",
"instantiation_dict": {
"address": "155 5th Street, San Francisco, CA"
},
"intent": "Change the delivery address for my most recent order to 155 5th Street, San Francisco, CA.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The shipping address cannot be changed after an order is placed in the website.",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 191
},
{
"sites": [
"shopping"
],
"task_id": 797,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Change the delivery address for my most recent order to {{address}}.",
"instantiation_dict": {
"address": "3 Oxford St, Cambridge, MA"
},
"intent": "Change the delivery address for my most recent order to 3 Oxford St, Cambridge, MA.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The shipping address cannot be changed after an order is placed in the website.",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 191
},
{
"sites": [
"shopping"
],
"task_id": 798,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Change the delivery address for my most recent order to {{address}}.",
"instantiation_dict": {
"address": "77 Massachusetts Ave, Cambridge, MA"
},
"intent": "Change the delivery address for my most recent order to 77 Massachusetts Ave, Cambridge, MA.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The shipping address cannot be changed after an order is placed in the website.",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 191
},
{
"sites": [
"gitlab"
],
"task_id": 799,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a new group \"{{name}}\" with members {{members}}",
"instantiation_dict": {
"name": "n-lab",
"members": "patou, egpast, westurner, jontutcher"
},
"intent": "create a new group \"n-lab\" with members patou, egpast, westurner, jontutcher",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/groups/n-lab/-/group_members",
"locator": "",
"required_contents": {
"must_include": [
"@patou",
"@egpast",
"@westurner",
"@jontutcher"
]
}
}
]
},
"intent_template_id": 600
},
{
"sites": [
"gitlab"
],
"task_id": 800,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a new group \"{{name}}\" with members {{members}}",
"instantiation_dict": {
"name": "x-lab",
"members": "JonasVautherin, dilipchandima, dawiss1337, bmyun, DCMJY"
},
"intent": "create a new group \"x-lab\" with members JonasVautherin, dilipchandima, dawiss1337, bmyun, DCMJY",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/groups/x-lab/-/group_members",
"locator": "",
"required_contents": {
"must_include": [
"@JonasVautherin",
"@dilipchandima",
"@dawiss1337",
"@bmyun",
"@DCMJY"
]
}
}
]
},
"intent_template_id": 600
},
{
"sites": [
"gitlab"
],
"task_id": 801,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a new group \"{{name}}\" with members {{members}}",
"instantiation_dict": {
"name": "crew",
"members": "ASWATFZLLC, patrickhlauke, westurner, linkmatrix"
},
"intent": "create a new group \"crew\" with members ASWATFZLLC, patrickhlauke, westurner, linkmatrix",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/groups/crew/-/group_members",
"locator": "",
"required_contents": {
"must_include": [
"@ASWATFZLLC",
"@patrickhlauke",
"@westurner",
"@linkmatrix"
]
}
}
]
},
"intent_template_id": 600
},
{
"sites": [
"gitlab"
],
"task_id": 802,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a new group \"{{name}}\" with members {{members}}",
"instantiation_dict": {
"name": "coding_friends",
"members": "qhduan, Agnes-U"
},
"intent": "create a new group \"coding_friends\" with members qhduan, Agnes-U",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/groups/coding_friends/-/group_members",
"locator": "",
"required_contents": {
"must_include": [
"@qhduan",
"@Agnes-U"
]
}
}
]
},
"intent_template_id": 600
},
{
"sites": [
"gitlab"
],
"task_id": 803,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "create a new group \"{{name}}\" with members {{members}}",
"instantiation_dict": {
"name": "webagent",
"members": "pandey2000, sayakpaul, sayakpaul"
},
"intent": "create a new group \"webagent\" with members pandey2000, sayakpaul, sayakpaul",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/groups/webagent/-/group_members",
"locator": "",
"required_contents": {
"must_include": [
"@pandey2000",
"@sayakpaul"
]
}
}
]
},
"intent_template_id": 600
},
{
"sites": [
"gitlab"
],
"task_id": 804,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} to {{account}}.",
"instantiation_dict": {
"issue": "flash alerts",
"account": "myself and primer"
},
"intent": "Assign the issue regarding flash alerts to myself and primer.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
"locator": "",
"required_contents": {
"must_include": [
"Add documentation on using Flash alerts in dialog components",
"Clarify usage of flash alert"
]
}
},
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=primer",
"locator": "",
"required_contents": {
"must_include": [
"Add documentation on using Flash alerts in dialog components",
"Clarify usage of flash alert"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 805,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "Submit a merge request for {{source_branch}} to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "the branch that implements the support of template strings",
"target_branch": "master",
"reviewer": "myself and Roshan"
},
"intent": "Submit a merge request for the branch that implements the support of template strings to be merged into master branch, assign myself and Roshan as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "feature/replace-gulp"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "main"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Byte Blaze",
"Roshan Jossy"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 806,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "a11yproject.com/redesign",
"target_branch": "markdown-figure-block",
"reviewer": "myself"
},
"intent": "Submit a merge request for a11yproject.com/redesign branch to be merged into markdown-figure-block branch, assign myself as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "redesign"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "feature/markdown-figure-block"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Byte Blaze"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 807,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
"instantiation_dict": {
"source_branch": "build time debug",
"target_branch": "main",
"reviewer": "myself"
},
"intent": "Submit a merge request for build time debug branch to be merged into main branch, assign myself as the reviewer",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/merge_requests",
"program_html": [
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
"required_contents": {
"exact_match": "main"
}
},
{
"url": "last",
"locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
"required_contents": {
"exact_match": "debug-build-time"
}
},
{
"url": "last",
"locator": "document.querySelector('.block.reviewer').outerText",
"required_contents": {
"must_include": [
"Byte Blaze"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 335
},
{
"sites": [
"gitlab"
],
"task_id": 808,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
"instantiation_dict": {
"repo": "cloud-to-butt",
"issue": "Let's keep the project alive",
"account": "myself",
"due": "the end of Q1 2033"
},
"intent": "Create an issue in cloud-to-butt repo with title \"Let's keep the project alive\". Assign the issue to myself. Set due date to be the end of Q1 2033",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/cloud-to-butt/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
"required_contents": {
"exact_match": "Let's keep the project alive"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
"required_contents": {
"must_include": [
"Mar 31, 2033"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.assignee').outerText",
"required_contents": {
"must_include": [
"Byte Blaze"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 327
},
{
"sites": [
"gitlab"
],
"task_id": 809,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
"instantiation_dict": {
"repo": "a11yproject",
"issue": "404 for many URLs",
"account": "myself",
"due": "2030-1-3"
},
"intent": "Create an issue in a11yproject repo with title \"404 for many URLs\". Assign the issue to myself. Set due date to be 2030-1-3",
"require_reset": false,
"eval": {
"eval_types": [
"url_match",
"program_html"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
"required_contents": {
"exact_match": "404 for many URLs"
}
},
{
"url": "last",
"locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
"required_contents": {
"must_include": [
"Jan 3, 2030"
]
}
},
{
"url": "last",
"locator": "document.querySelector('.block.assignee').outerText",
"required_contents": {
"must_include": [
"Byte Blaze"
]
}
}
],
"url_note": "GOLD in PRED"
},
"intent_template_id": 327
},
{
"sites": [
"gitlab"
],
"task_id": 810,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} repo to {{account}}.",
"instantiation_dict": {
"repo": "primer design guide",
"issue": "flash alert",
"account": "myself"
},
"intent": "Assign the issue regarding flash alert in primer design guide repo to myself.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
"locator": "",
"required_contents": {
"must_include": [
"Add documentation on using Flash alerts in dialog components",
"Clarify usage of flash alert"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 811,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
"instantiation_dict": {
"repo": "a11yproject",
"issue": 404,
"account": "myself"
},
"intent": "Assign the issue regarding 404 in a11yproject to myself.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
"locator": "",
"required_contents": {
"must_include": [
"404s, bad host, timeouts, bad urls for URLs linked from website"
]
}
}
]
},
"intent_template_id": 999
}
]
Reference in New Issue View Git Blame Copy Permalink