yuyr/AgentOccam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
AgentOccam/config_files/test.raw.json
Yao Liu c078ba6292 update
2025-01-28 14:46:28 -08:00

29625 lines
862 KiB
JSON
Raw Permalink Blame History

[
{
"sites": [
"shopping_admin"
],
"task_id": 0,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 1,
"year": 2022
},
"intent": "What is the top-1 best-selling product in 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Quest Lumaflex\u2122 Band"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 1,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling brand in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Quarter 1 2022"
},
"intent": "What is the top-1 best-selling brand in Quarter 1 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Sprite"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sprite"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 2,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Quarter 1 2022"
},
"intent": "What is the top-1 best-selling product type in Quarter 1 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Yoga ball"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yoga ball"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 3,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 2,
"year": 2022
},
"intent": "What are the top-2 best-selling product in 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Quest Lumaflex\u2122 Band",
"Sprite Stasis Ball 65 cm"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band, Sprite Stasis Ball 65 cm"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 4,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{period}}",
"instantiation_dict": {
"n": 3,
"period": "Jan 2023"
},
"intent": "What are the top-3 best-selling product in Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Impulse Duffle",
"Overnight Duffle",
"Hawkeye Yoga Short-32-Blue"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Impulse Duffle, Overnight Duffle, Hawkeye Yoga Short-32-Blue"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 5,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
"instantiation_dict": {
"n": 1,
"period": "Jan 2023"
},
"intent": "What is the top-1 best-selling product type in Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Duffle"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Duffle"
},
"intent_template_id": 279
},
{
"sites": [
"shopping_admin"
],
"task_id": 6,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the top-{{n}} best-selling product in {{year}}",
"instantiation_dict": {
"n": 5,
"year": 2023
},
"intent": "What are the top-5 best-selling product in 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sprite Yoga Strap 6 foot",
"Overnight Duffle",
"Ida Workout Parachute Pant-29-Purple",
"Hawkeye Yoga Short-32-Blue",
"Sprite Stasis Ball 65 cm"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sprite Yoga Strap 6 foot, Overnight Duffle, Ida Workout Parachute Pant-29-Purple, Hawkeye Yoga Short-32-Blue, Sprite Stasis Ball 65 cm"
},
"intent_template_id": 279
},
{
"sites": [
"map"
],
"task_id": 7,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Mellon University",
"radius": "50 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 50 km to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 8,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Mellon University",
"radius": "5 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 5 km to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"There is no airport within 5 km of Carnegie Mellon University"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "There is no airport within 5 km of Carnegie Mellon University"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 9,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "international airports",
"start": "Carnegie Art Museum",
"radius": "30 km"
},
"intent": "Tell me the full address of all international airports that are within a driving distance of 30 km to Carnegie Art Museum",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
},
"intent_template_id": 79
},
{
"sites": [
"map"
],
"task_id": 10,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
"instantiation_dict": {
"airport_type": "US international airports",
"start": "Niagara Falls",
"radius": "60 km"
},
"intent": "Tell me the full address of all US international airports that are within a driving distance of 60 km to Niagara Falls",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States",
"Buffalo-Niagara International Airport, Holtz Drive, Town of Cheektowaga, Erie County, New York, 14225, United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States Buffalo-Niagara International Airport, South Youngs Road, Town of Cheektowaga, Erie County, New York, 14221, United States"
},
"intent_template_id": 79
},
{
"sites": [
"shopping_admin"
],
"task_id": 11,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "disappointed"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"disappointed\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"6"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "6"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 12,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "satisfied"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"satisfied\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 13,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "decent"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"decent\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 14,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "not useful"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"not useful\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 288
},
{
"sites": [
"shopping_admin"
],
"task_id": 15,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
"instantiation_dict": {
"term": "best"
},
"intent": "Tell me the the number of reviews that our store received by far that mention term \"best\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 288
},
{
"sites": [
"map"
],
"task_id": 16,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "5000 Fifth Avenue, Pittsburgh",
"end": "UPMC family health center"
},
"intent": "Compare the time for walking and driving route from 5000 Fifth Avenue, Pittsburgh to UPMC family health center",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 2min; walking: 16min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Driving: 2min. Walking: 16min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 17,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "AMC Waterfront",
"end": "Carnegie Mellon University"
},
"intent": "Compare the time for walking and driving route from AMC Waterfront to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 13min; walking: 1h 35min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 13min, walking: 1h 35min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 18,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "AMC Waterfront",
"end": "Univ of Pittsburgh"
},
"intent": "Compare the time for walking and driving route from AMC Waterfront to Univ of Pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 15min; walking: 1h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 15min, walking: 1h 47min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 19,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "Carnegie Science Center",
"end": "Carnegie Mellon University"
},
"intent": "Compare the time for walking and driving route from Carnegie Science Center to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 12min; walking: 1h 44min."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 12min, walking: 1h 44min."
},
"intent_template_id": 73
},
{
"sites": [
"map"
],
"task_id": 20,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Compare the difference in time for walking and driving route from {{start}} to {{end}}",
"instantiation_dict": {
"start": "Randyland",
"end": "Carnegie Mellon University"
},
"intent": "Compare the difference in time for walking and driving route from Randyland to Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"driving: 13min; walking: 1h 45min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "driving: 13min, walking: 1h 45min."
},
"intent_template_id": 73
},
{
"sites": [
"shopping"
],
"task_id": 21,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/6s-wireless-headphones-over-ear-noise-canceling-hi-fi-bass-foldable-stereo-wireless-kid-headsets-earbuds-with-built-in-mic-micro-sd-tf-fm-for-iphone-samsung-ipad-pc-black-gold.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "ear cups being small"
},
"intent": "List out reviewers, if exist, who mention about ear cups being small",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Joseph Brzezinski",
"Catso",
"Dibbins",
"Anglebert Dinkherhump",
"Michelle Davis"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Joseph Brzezinski, Catso, Dibbins, Anglebert Dinkherhump, Michelle Davis"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 22,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/fujifilm-finepix-z200fd-10mp-digital-camera-with-5x-optical-dual-image-stabilized-zoom-black.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "under water photo"
},
"intent": "List out reviewers, if exist, who mention about under water photo",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no review about under water photo",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 23,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/3-pack-samsung-galaxy-s6-screen-protector-nearpow-tempered-glass-screen-protector-with-9h-hardness-crystal-clear-easy-bubble-free-installation-scratch-resist.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "good fingerprint resistant"
},
"intent": "List out reviewers, if exist, who mention about good fingerprint resistant",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rachel",
"T. Gannon"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rachel, T. Gannon, "
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 24,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/haflinger-men-s-wool-felt-open-back-slippers-beige-550-peat-us-7.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "price being unfair"
},
"intent": "List out reviewers, if exist, who mention about price being unfair",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no reivew about price being unfair",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 25,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "average print quality"
},
"intent": "List out reviewers, if exist, who mention about average print quality",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Goldfish",
"Roxanne Brandon Coffey"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "GoldfishGoldfish, Roxanne Brandon Coffey"
},
"intent_template_id": 222
},
{
"sites": [
"shopping"
],
"task_id": 26,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
"geolocation": null,
"intent_template": "List out reviewers, if exist, who mention about {{description}}",
"instantiation_dict": {
"description": "complain of the customer service"
},
"intent": "List out reviewers, if exist, who mention about complain of the customer service",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Bob in Vegas",
"RemyR"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Bob in Vegas, RemyRRemyR"
},
"intent_template_id": 222
},
{
"sites": [
"reddit"
],
"task_id": 27,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "Showerthoughts"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Showerthoughts forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 28,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "Worcester"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Worcester forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 29,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "DIY"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the DIY forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 30,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "space"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the space forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"reddit"
],
"task_id": 31,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
"instantiation_dict": {
"forum": "photoshopbattles"
},
"intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the photoshopbattles forum.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 33
},
{
"sites": [
"map"
],
"task_id": 32,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"information": "the walking distance",
"target1": "Hilton hotel",
"target2": "the nearest supermarket own by a local company"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the walking distance to the nearest supermarket own by a local company from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"DoubleTree by Hilton Hotel Pittsburgh Airport",
"2.0km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 2.0km"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 33,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hilton hotel",
"information": "the shortest walking distance",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the shortest walking distance to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"DoubleTree by Hilton Hotel Pittsburgh Airport",
"1.4km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 1.4km"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 34,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hyatt hotel",
"information": "the shortest walking time",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the shortest walking time to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport"
],
"fuzzy_match": [
"Time: 3h 30min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport\n3:30"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 35,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
"instantiation_dict": {
"place": "Pittsburgh Airport",
"target1": "Hyatt hotel",
"information": "the minimal driving time",
"target2": "a supermarket"
},
"intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the minimal driving time to a supermarket from the hotel.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport"
],
"fuzzy_match": [
"Time: 15min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Time: 15min"
},
"intent_template_id": 78
},
{
"sites": [
"map"
],
"task_id": 36,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "social security administration",
"location": "Carnegie Mellon University"
},
"intent": "Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 37,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "police station",
"location": "gates building at CMU"
},
"intent": "Check if the police station in pittsburgh can be reached in one hour by car from gates building at CMU",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 38,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "duquesne university",
"location": "pittsburgh airport"
},
"intent": "Check if the duquesne university in pittsburgh can be reached in one hour by car from pittsburgh airport",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 39,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "walmart",
"location": "5600 fifth avenue"
},
"intent": "Check if the walmart in pittsburgh can be reached in one hour by car from 5600 fifth avenue",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"map"
],
"task_id": 40,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
"instantiation_dict": {
"place": "amc theatre",
"location": "hobart street"
},
"intent": "Check if the amc theatre in pittsburgh can be reached in one hour by car from hobart street",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Yes"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Yes"
},
"intent_template_id": 77
},
{
"sites": [
"shopping_admin"
],
"task_id": 41,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "1"
},
"intent": "List the top 1 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "hollister"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister"
},
"intent_template_id": 285
},
{
"sites": [
"shopping_admin"
],
"task_id": 42,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "2"
},
"intent": "List the top 2 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"hollister",
"Joust Bag"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister, Joust Bag"
},
"intent_template_id": 285
},
{
"sites": [
"shopping_admin"
],
"task_id": 43,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "List the top {{n}} search terms in my store",
"instantiation_dict": {
"n": "3"
},
"intent": "List the top 3 search terms in my store",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"hollister",
"Joust Bag",
"Antonia Racer Tank"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hollister, Joust Bag, Antonia Race Tank"
},
"intent_template_id": 285
},
{
"sites": [
"gitlab"
],
"task_id": 44,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Check out my todos",
"instantiation_dict": {},
"intent": "Check out my todos",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/todos",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 303
},
{
"sites": [
"gitlab"
],
"task_id": 45,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "Check out the most recent open issues",
"instantiation_dict": {},
"intent": "Check out the most recent open issues",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?sort=created_asc&state=opened",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 300
},
{
"sites": [
"gitlab"
],
"task_id": 46,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "Check out the most recent open issues",
"instantiation_dict": {},
"intent": "Check out the most recent open issues",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/issues/?sort=created_date&state=opened",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 300
},
{
"sites": [
"shopping"
],
"task_id": 47,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"0 order",
"$0 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0 order, $0 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 48,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past three days"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past three days, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"0 order",
"$0 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0 order, $0 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 49,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past four month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past four month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3 orders",
"$845.49 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3 orders, $845.49 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 50,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past year"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past year, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"24 orders",
"$6560.69 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "24 orders, $6560.69 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"shopping"
],
"task_id": 51,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
"instantiation_dict": {
"period": "over the past six month"
},
"intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past six month, and the total amount of money I spent.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"12 orders",
"$1603.69 total spend"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "12 orders, $1603.69 total spend"
},
"intent_template_id": 197
},
{
"sites": [
"map"
],
"task_id": 52,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Mellon University",
"end": "starbucks on Craig Street"
},
"intent": "How long does it take to walk from Carnegie Mellon University to starbucks on Craig Street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"7 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "7 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 53,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Univ of Pittsburgh",
"end": "starbucks on Craig Street"
},
"intent": "How long does it take to walk from Univ of Pittsburgh to starbucks on Craig Street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"18 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "18 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 54,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Mellon University",
"end": "Univ of Pittsburgh"
},
"intent": "How long does it take to walk from Carnegie Mellon University to Univ of Pittsburgh?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"25 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "25 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 55,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "the starbuck near CMU",
"end": "Chatham university"
},
"intent": "How long does it take to walk from the starbuck near CMU to Chatham university?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"30 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "30 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 56,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How long does it take to walk from {{start}} to {{end}}?",
"instantiation_dict": {
"start": "Carnegie Museum of Art",
"end": "a library at CMU"
},
"intent": "How long does it take to walk from Carnegie Museum of Art to a library at CMU?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"11 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "11 min"
},
"intent_template_id": 68
},
{
"sites": [
"map"
],
"task_id": 57,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "university center at Carnegie Mellon University"
},
"intent": "Tell me the closest restaurant(s) to university center at Carnegie Mellon University",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"El Gallo de Oro",
"Back Bar Grill",
"Grano",
"Beefsteak",
"Nourish",
"Schatz Dining Room",
"Au Bon Pain"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "El Gallo de Oro, Back Bar Grill, Grano, Beefsteak, Nourish, Schatz Dining Room, Au Bon Pain"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 58,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "cafe",
"place2": "CMU Hunt library"
},
"intent": "Tell me the closest cafe(s) to CMU Hunt library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"De Fer Coffee & Tea"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "De Fer Coffee & Tea"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 59,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Hunt library"
},
"intent": "Tell me the closest restaurant(s) to CMU Hunt library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The exchange"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The exchange"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 60,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Posner Hall"
},
"intent": "Tell me the closest restaurant(s) to CMU Posner Hall",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The exchange"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The exchange"
},
"intent_template_id": 69
},
{
"sites": [
"map"
],
"task_id": 61,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
"instantiation_dict": {
"place1": "restaurant",
"place2": "CMU Sorrells Library"
},
"intent": "Tell me the closest restaurant(s) to CMU Sorrells Library",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"La Prima Espresso"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "La Prima Espresso"
},
"intent_template_id": 69
},
{
"sites": [
"shopping_admin"
],
"task_id": 62,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "most"
},
"intent": "Which customer has completed the most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jane Smith"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jane Smith"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 63,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer(s) has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "second most"
},
"intent": "Which customer(s) has completed the second most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Adam Garcia",
"Michael Nguyen",
"Sarah Miller"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Adam Garcia, Michael Nguyen, Sarah Miller"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 64,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has placed {{number}} orders in the entire history?",
"instantiation_dict": {
"number": "2"
},
"intent": "Which customer has placed 2 orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Kim",
"Lisa Green",
"Julia Williams",
"Brian Smith",
"Alexander Thomas"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Kim, Lisa Green, Julia Williams, Brian Smith, Alexander Thomas"
},
"intent_template_id": 276
},
{
"sites": [
"shopping_admin"
],
"task_id": 65,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
"instantiation_dict": {
"quantifier": "fifth most"
},
"intent": "Which customer has completed the fifth most number of orders in the entire history?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jane Doe"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jane Doe"
},
"intent_template_id": 276
},
{
"sites": [
"reddit"
],
"task_id": 66,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the post URLs that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the post URLs that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"http://www.reddit.com/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol",
"http://www.reddit.com/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "__REDDIT__/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol, __REDDIT__/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 67,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the book names from posts that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the book names from posts that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"A Christmas Carol",
"The Hobbit"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "A Christmas Carol, The Hobbit"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 68,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "show me the author name and the book name from posts that recommand a single book"
},
"intent": "Among the top 10 post in \"books\" forum, show me the author name and the book name from posts that recommand a single book",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"A Christmas Carol",
"Levar Burton",
"The Hobbit",
"J. R. R. Tolkien"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "A Christmas Carol by Levar Burton: , The Hobbit by J. R. R. Tolkien"
},
"intent_template_id": 17
},
{
"sites": [
"reddit"
],
"task_id": 69,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
"instantiation_dict": {
"number": 10,
"subreddit": "books",
"description": "is there any post talks about supporting local book stores? If so, tell me the organizations involved"
},
"intent": "Among the top 10 post in \"books\" forum, is there any post talks about supporting local book stores? If so, tell me the organizations involved",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"bookshop.org"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "bookshop.org"
},
"intent_template_id": 17
},
{
"sites": [
"map"
],
"task_id": 70,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Carnegie Mellon University"
},
"intent": "What is the zip code of Carnegie Mellon University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "15213"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "15213"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 71,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Chatham University"
},
"intent": "What is the zip code of Chatham University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "15232"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "15232"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 72,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Yale University"
},
"intent": "What is the zip code of Yale University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "06516"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "06516"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 73,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the zip code of {{place}}?",
"instantiation_dict": {
"place": "Columbia University"
},
"intent": "What is the zip code of Columbia University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "10027"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "10027"
},
"intent_template_id": 70
},
{
"sites": [
"map"
],
"task_id": 74,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Carnegie Mellon University",
"apple store shadyside",
"starbucks on craig street"
]
},
"intent": "Given the following locations, ['Carnegie Mellon University', 'apple store shadyside', 'starbucks on craig street'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The order is Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
},
"intent_template_id": 65
},
{
"sites": [
"map"
],
"task_id": 75,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Massachusetts Institute of Technology",
"Harvard University",
"Boston Logan International Airport"
]
},
"intent": "Given the following locations, ['Massachusetts Institute of Technology', 'Harvard University', 'Boston Logan International Airport'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The order is Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
},
"intent_template_id": 65
},
{
"sites": [
"map"
],
"task_id": 76,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"instantiation_dict": {
"place_list": [
"Princeton University",
"Yale University",
"Harvard University"
]
},
"intent": "Given the following locations, ['Princeton University', 'Yale University', 'Harvard University'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The order is Princeton University, Yale University, Harvard University"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Princeton University, Yale University, Harvard University"
},
"intent_template_id": 65
},
{
"sites": [
"shopping_admin"
],
"task_id": 77,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Pending"
},
"intent": "What is the total count of Pending reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5"
},
"intent_template_id": 277
},
{
"sites": [
"shopping_admin"
],
"task_id": 78,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Approved"
},
"intent": "What is the total count of Approved reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"346"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "346"
},
"intent_template_id": 277
},
{
"sites": [
"shopping_admin"
],
"task_id": 79,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
"instantiation_dict": {
"status": "Not Approved"
},
"intent": "What is the total count of Not Approved reviews amongst all the reviews?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 277
},
{
"sites": [
"map"
],
"task_id": 80,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Carnegie Mellon University",
"place_B": "Starbucks on Craig Street",
"place_C": "Pittsburgh International Airport"
},
"intent": "What is the duration required to first walk from Carnegie Mellon University to Starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"38 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "38 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 81,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Univ of Pittsburgh",
"place_B": "starbucks on Craig Street",
"place_C": "Pittsburgh International Airport"
},
"intent": "What is the duration required to first walk from Univ of Pittsburgh to starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"49 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "49 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 82,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Massachusetts Institute of Technology",
"place_B": "Harvard University",
"place_C": "Boston Logan International Airport"
},
"intent": "What is the duration required to first walk from Massachusetts Institute of Technology to Harvard University, and then drive to Boston Logan International Airport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"63 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "63 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 83,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
"instantiation_dict": {
"place_A": "Carnegie Mellon University",
"place_B": "apple store shadyside",
"place_C": "starbucks on craig street"
},
"intent": "What is the duration required to first walk from Carnegie Mellon University to apple store shadyside, and then drive to starbucks on craig street?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"22 min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "22 min"
},
"intent_template_id": 72
},
{
"sites": [
"map"
],
"task_id": 84,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "DoubleTree by Hilton New York Downtown",
"place": "Keens Steakhouse"
},
"intent": "From my stay at DoubleTree by Hilton New York Downtown, what's the estimated driving time to reach Keens Steakhouse?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"14 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "14 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 85,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "La Quinta Inn near the airport",
"place": "Carnegie Mellon University"
},
"intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Carnegie Mellon University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"30 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "30 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 86,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "La Quinta Inn near the airport",
"place": "Upitt"
},
"intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Upitt?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"29 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "29 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 87,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "red roof inn",
"place": "Pittsburgh science museum"
},
"intent": "From my stay at red roof inn, what's the estimated driving time to reach Pittsburgh science museum?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"20 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "20 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 88,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
"instantiation_dict": {
"hotel": "Homewood Suites Southpointe",
"place": "PPG Paints Arena"
},
"intent": "From my stay at Homewood Suites Southpointe, what's the estimated driving time to reach PPG Paints Arena?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"34 minutes"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "34 minutes"
},
"intent_template_id": 64
},
{
"sites": [
"map"
],
"task_id": 89,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Connecticut"
},
"intent": "Which US states border Connecticut?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rhode Island",
"Massachusetts",
"New York"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rhode Island, Massachusetts, New York"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 90,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Pennsylvania"
},
"intent": "Which US states border Pennsylvania?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Ohio",
"Maryland",
"New York",
"New Jersey",
"Delaware",
"West Virginia"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Ohio, Maryland, New York, New Jersey, Delaware, West Virginia"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 91,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Massachusetts"
},
"intent": "Which US states border Massachusetts?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Rhode Island",
"Connecticut",
"New York",
"New Hampshire",
"Vermont"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rhode Island, Connecticut, New York, New Hampshire, Vermont"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 92,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "Vermont"
},
"intent": "Which US states border Vermont?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"New York",
"New Hampshire",
"Massachusetts"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "New York, New Hampshire, Massachusetts"
},
"intent_template_id": 67
},
{
"sites": [
"map"
],
"task_id": 93,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Which US states border {{state}}?",
"instantiation_dict": {
"state": "New Hampshire"
},
"intent": "Which US states border New Hampshire?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Massachusetts",
"Vermont",
"Maine"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Massachusetts, Vermont, Maine"
},
"intent_template_id": 67
},
{
"sites": [
"shopping_admin"
],
"task_id": 94,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Telll me the grand total of invoice {{id}}.",
"instantiation_dict": {
"id": "000000001"
},
"intent": "Telll me the grand total of invoice 000000001.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"36.39"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$36.39"
},
"intent_template_id": 274
},
{
"sites": [
"shopping_admin"
],
"task_id": 95,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Telll me the grand total of invoice {{id}}.",
"instantiation_dict": {
"id": "000000002"
},
"intent": "Telll me the grand total of invoice 000000002.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"39.64"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$39.64"
},
"intent_template_id": 274
},
{
"sites": [
"shopping"
],
"task_id": 96,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the status of my latest order and when will it arrive",
"instantiation_dict": {},
"intent": "Tell me the status of my latest order and when will it arrive",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"The last order was canceled. It will never arrive."
]
},
"reference_url": "",
"program_html": [],
"reference_answer_raw_annotation": "The last order was canceled. It will never arrive.",
"string_note": ""
},
"intent_template_id": 193
},
{
"sites": [
"map",
"wikipedia"
],
"task_id": 97,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
"instantiation_dict": {},
"intent": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"914km |OR| 914 km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "914 km"
},
"intent_template_id": 120
},
{
"sites": [
"map"
],
"task_id": 98,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "tea cafe",
"start": "University of Pittsburgh"
},
"intent": "Where is the nearest tea cafe to University of Pittsburgh, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Fuku Tea",
"3716",
"Forbes Avenue",
"Central Oakland",
"Pittsburgh",
"653m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Fuku Tea, 3716, Forbes Avenue, Oakland, Central Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n653m"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 99,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "Five Guys",
"start": "5700 Penn Ave"
},
"intent": "Where is the nearest Five Guys to 5700 Penn Ave, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Five Guys",
"117",
"South Bouquet Street",
"North Oakland",
"Pittsburgh",
"4.0km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Five Guys, 117, South Bouquet Street, Oakland, North Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n4.0km"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 100,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "Starbucks",
"start": "Carnegie Mellon"
},
"intent": "Where is the nearest Starbucks to Carnegie Mellon, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Starbucks",
"417",
"South Craig Street",
"Bellefield",
"Pittsburgh",
"557m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Starbucks, 417, South Craig Street, Bellefield, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n557m"
},
"intent_template_id": 66
},
{
"sites": [
"map"
],
"task_id": 101,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
"instantiation_dict": {
"places": "In-N-Out",
"start": "Upitts"
},
"intent": "Where is the nearest In-N-Out to Upitts, and what is the walking distance to it?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no In-N-Out near University of Pittsburgh",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 66
},
{
"sites": [
"gitlab"
],
"task_id": 102,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "help needed",
"repo": "a11yproject/a11yproject.com"
},
"intent": "Display the list of issues in the a11yproject/a11yproject.com repository that have labels related to help needed",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/?label_name%5B%5D=help%20wanted",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 103,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "questions",
"repo": "kkroening/ffmpeg-python"
},
"intent": "Display the list of issues in the kkroening/ffmpeg-python repository that have labels related to questions",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/kkroening/ffmpeg-python/-/issues/?label_name%5B%5D=question",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 104,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "flaky-test",
"repo": "keycloak/keycloak"
},
"intent": "Display the list of issues in the keycloak/keycloak repository that have labels related to flaky-test",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/keycloak/keycloak/-/issues/?label_name%5B%5D=flaky-test",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 105,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "OpenAPI Generator CLI",
"repo": "OpenAPITools/openapi-generator"
},
"intent": "Display the list of issues in the OpenAPITools/openapi-generator repository that have labels related to OpenAPI Generator CLI",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/OpenAPITools/openapi-generator/-/issues/?label_name%5B%5D=OpenAPI%20Generator%20CLI",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"gitlab"
],
"task_id": 106,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
"instantiation_dict": {
"label": "BUG",
"repo": "umano/AndroidSlidingUpPanel"
},
"intent": "Display the list of issues in the umano/AndroidSlidingUpPanel repository that have labels related to BUG",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/umano/AndroidSlidingUpPanel/-/issues/?label_name%5B%5D=BUG",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 349
},
{
"sites": [
"shopping_admin"
],
"task_id": 107,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from May to December 2022"
},
"intent": "Presents the monthly count of successful orders from May to December 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"October: 4 orders",
"November: 5 orders",
"December: 10 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 108,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "01/2023-05/2023"
},
"intent": "Presents the monthly count of successful orders 01/2023-05/2023 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 12 orders",
"Feburary: 7 orders",
"March: 5 orders",
"April: 9 orders",
"May: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 12 orders Febulary: 7 orders March: 5 orders Apirl: 9 orders May: 5 orders"
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 109,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Jan to December 2022"
},
"intent": "Presents the monthly count of successful orders from Jan to December 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 11 orders",
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders",
"December: 10 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 110,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Jan to Nov 2022"
},
"intent": "Presents the monthly count of successful orders from Jan to Nov 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January: 11 orders",
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 111,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
"instantiation_dict": {
"period": "from Feb to Nov 2022"
},
"intent": "Presents the monthly count of successful orders from Feb to Nov 2022 in MM:COUNT format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Feburary: 16 orders",
"March: 14 orders",
"April: 7 orders",
"May: 8 orders",
"June: 13 orders",
"July: 9 orders",
"August: 8 orders",
"Sepetember: 10 orders",
"Octorbor: 4 orders",
"November: 5 orders"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
},
"intent_template_id": 270
},
{
"sites": [
"shopping_admin"
],
"task_id": 112,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Circe fleece"
},
"intent": "Show me the customers who have expressed dissatisfaction with Circe fleece?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Hannah Lim"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hannah Lim"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 113,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Olivia zip jacket"
},
"intent": "Show me the customers who have expressed dissatisfaction with Olivia zip jacket?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Emma Lopez",
"Seam Miller"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Emma Lopez, Seam Miller"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 114,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "Antonia racer tank"
},
"intent": "Show me the customers who have expressed dissatisfaction with Antonia racer tank?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Shaunte",
"Merrie"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shaunte, Merrie"
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 115,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}",
"instantiation_dict": {
"product": "Chloe tank"
},
"intent": "Show me the name of the customers who have expressed dissatisfaction with Chloe tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no negative review for Chloe tank",
"reference_answer_raw_annotation": ""
},
"intent_template_id": 245
},
{
"sites": [
"shopping_admin"
],
"task_id": 116,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}?",
"instantiation_dict": {
"product": "tanks products"
},
"intent": "Show me the name of the customers who have expressed dissatisfaction with tanks products?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Alexander",
"Carma",
"Dominic",
"Merrie",
"Monroe",
"Scotty",
"Shaunte",
"Teofila",
"Valorie"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Alexander, Carma, Dominic, Merrie, Monroe, Scotty, Shaunte, Teofila, Valorie"
},
"intent_template_id": 245
},
{
"sites": [
"shopping"
],
"task_id": 117,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the date when I made my first purchase on this site?",
"instantiation_dict": {},
"intent": "What is the date when I made my first purchase on this site?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3/2/22"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3/2/22"
},
"intent_template_id": 161
},
{
"sites": [
"shopping"
],
"task_id": 118,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have jaw bruxism problem, show me something that could alleviate the problem.",
"instantiation_dict": {},
"intent": "I have jaw bruxism problem, show me something that could alleviate the problem.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "",
"required_contents": {
"must_include": [
"jaw",
"bruxism",
"mouth guard"
]
}
}
]
},
"intent_template_id": 151
},
{
"sites": [
"shopping_admin"
],
"task_id": 119,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "Tell me the reasons why customers like Antonia Racer Tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Its color and style is good"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Its color and style is good"
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 120,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Ana Running Short"
},
"intent": "Tell me the reasons why customers like Ana Running Short",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"It is comfortable"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "It is comfortable"
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 121,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Circe hooded fleece"
},
"intent": "Tell me the reasons why customers like Circe hooded fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Warm and comfortable. True to size."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Warm and comfortable. True to size."
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 122,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Olivia zip jacket"
},
"intent": "Tell me the reasons why customers like Olivia zip jacket",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Lightweight, comfortable and stylish. Good design and details."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lightweight, comfortable, and stylish. Good design and details."
},
"intent_template_id": 250
},
{
"sites": [
"shopping_admin"
],
"task_id": 123,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the reasons why customers like {{product}}",
"instantiation_dict": {
"product": "Circe's products"
},
"intent": "Tell me the reasons why customers like Circe's products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Warm and comfortable. True to size."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Warm and comfortable. True to size."
},
"intent_template_id": 250
},
{
"sites": [
"shopping"
],
"task_id": 124,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "wireless earphone"
},
"intent": "What is the price range of wireless earphone in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0.14",
"745.00"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$0.14 - $745.00"
},
"intent_template_id": 159
},
{
"sites": [
"shopping"
],
"task_id": 125,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "teeth grinding mouth guard"
},
"intent": "What is the price range of teeth grinding mouth guard in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1.46",
"85"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$1.46 - $85"
},
"intent_template_id": 159
},
{
"sites": [
"shopping"
],
"task_id": 126,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range of {{product}} in the One Stop Market?",
"instantiation_dict": {
"product": "Canon photo printer"
},
"intent": "What is the price range of Canon photo printer in the One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2.56",
"649.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$2.56 - $649.99"
},
"intent_template_id": 159
},
{
"sites": [
"shopping_admin"
],
"task_id": 127,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What brands appear most frequently among the top search terms?",
"instantiation_dict": {},
"intent": "What brands appear most frequently among the top search terms?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hollister",
"Joust",
"Antonia"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hollister, Joust, Antonia"
},
"intent_template_id": 1001
},
{
"sites": [
"shopping_admin"
],
"task_id": 128,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "2"
},
"intent": "What's the total number of items sold in the most recent 2 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 129,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "4"
},
"intent": "What's the total number of items sold in the most recent 4 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 130,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "5"
},
"intent": "What's the total number of items sold in the most recent 5 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"18"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "18"
},
"intent_template_id": 1002
},
{
"sites": [
"shopping_admin"
],
"task_id": 131,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
"instantiation_dict": {
"k": "7"
},
"intent": "What's the total number of items sold in the most recent 7 orders?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"25"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "25"
},
"intent_template_id": 1002
},
{
"sites": [
"gitlab"
],
"task_id": 132,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"repo": "a11yproject",
"date": "3/5/2023"
},
"intent": "How many commits did kilian make to a11yproject on 3/5/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 133,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Eric",
"repo": "a11yproject",
"date": "3/2"
},
"intent": "How many commits did Eric make to a11yproject on 3/2?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 134,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"repo": "a11yproject",
"date": "3/1/2023"
},
"intent": "How many commits did kilian make to a11yproject on 3/1/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 135,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Eric and Kilian",
"repo": "a11yproject",
"date": "1/3/2023"
},
"intent": "How many commits did Eric and Kilian make to a11yproject on 1/3/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 322
},
{
"sites": [
"gitlab"
],
"task_id": 136,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
"instantiation_dict": {
"user": "Steven Woodson",
"repo": "a11y-webring.club",
"date": "2/6/2023"
},
"intent": "How many commits did Steven Woodson make to a11y-webring.club on 2/6/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5"
},
"intent_template_id": 322
},
{
"sites": [
"map"
],
"task_id": 137,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the city where the Liberty Bell is located",
"city2": "the home city of Pirates"
},
"intent": "What is the estimated driving time between the city where the Liberty Bell is located and the home city of Pirates?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"5h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5h 47min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 138,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the big apple",
"city2": "the city with the most authentic Philly cheesesteaks"
},
"intent": "What is the estimated driving time between the big apple and the city with the most authentic Philly cheesesteaks?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"1h 58min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1h 58min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 139,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the hometown of Joe Biden",
"city2": "Bridgeport"
},
"intent": "What is the estimated driving time between the hometown of Joe Biden and Bridgeport?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3h 20min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3h 20min"
},
"intent_template_id": 51
},
{
"sites": [
"map"
],
"task_id": 140,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
"instantiation_dict": {
"city1": "the city of Niagara Falls",
"city2": "the city of Yale University"
},
"intent": "What is the estimated driving time between the city of Niagara Falls and the city of Yale University?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"8h 33min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "8h 33min"
},
"intent_template_id": 51
},
{
"sites": [
"shopping"
],
"task_id": 141,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "food-related",
"time": "March 2023"
},
"intent": "How much I spent on food-related shopping during March 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"47.41"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$47.41"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 142,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "hair care and hair style",
"time": "Jan 2023"
},
"intent": "How much I spent on hair care and hair style shopping during Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"95.23"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$95.23"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 143,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "home decoration",
"time": "1/29/2023"
},
"intent": "How much I spent on home decoration shopping during 1/29/2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"265.69"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$265.69"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 144,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "food",
"time": "from mid Jan to the end Jan 2023"
},
"intent": "How much I spent on food shopping during from mid Jan to the end Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 145,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spent on {{category}} shopping during {{time}}",
"instantiation_dict": {
"category": "cooking and food",
"time": "March 2022"
},
"intent": "How much I spent on cooking and food shopping during March 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"52.35"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$52.35"
},
"intent_template_id": 162
},
{
"sites": [
"shopping"
],
"task_id": 146,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "size",
"product": "picture frame",
"time": "Sep 2022"
},
"intent": "What is the size configuration of the picture frame I bought Sep 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"16x24"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16x24"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 147,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "size",
"product": "picture frame",
"time": "2022"
},
"intent": "What is the size configuration of the picture frame I bought 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16x24"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16x24"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 148,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "color",
"product": "picture frame",
"time": "Sep 2022"
},
"intent": "What is the color configuration of the picture frame I bought Sep 2022",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Mist"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Mist"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 149,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "color",
"product": "artifical plants",
"time": "Feb 2023"
},
"intent": "What is the color configuration of the artifical plants I bought Feb 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Green-vines"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Green-vines"
},
"intent_template_id": 155
},
{
"sites": [
"shopping"
],
"task_id": 150,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
"instantiation_dict": {
"option": "price",
"product": "fake tree",
"time": "Jan 2023"
},
"intent": "What is the price configuration of the fake tree I bought Jan 2023",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"260.69"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "260.69"
},
"intent_template_id": 155
},
{
"sites": [
"map"
],
"task_id": 151,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "CMU",
"location2": "University of Pittsburgh"
},
"intent": "What is the minimum travel time by car from CMU to University of Pittsburgh?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 152,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "Schenley park",
"location2": "Upitt"
},
"intent": "What is the minimum travel time by car from Schenley park to Upitt?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 153,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "REI",
"location2": "CMU"
},
"intent": "What is the minimum travel time by car from REI to CMU?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"7min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "7min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 154,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "CMU gates building",
"location2": "Schenley park"
},
"intent": "What is the minimum travel time by car from CMU gates building to Schenley park?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 36
},
{
"sites": [
"map"
],
"task_id": 155,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
"instantiation_dict": {
"location1": "Animal Rescue League of Pittsburgh",
"location2": "Schenley park"
},
"intent": "What is the minimum travel time by car from Animal Rescue League of Pittsburgh to Schenley park?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"9min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9min"
},
"intent_template_id": 36
},
{
"sites": [
"gitlab"
],
"task_id": 156,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Checkout merge requests assigned to me",
"instantiation_dict": {},
"intent": "Checkout merge requests assigned to me",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/merge_requests?assignee_username=byteblaze",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 290
},
{
"sites": [
"shopping_admin"
],
"task_id": 157,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show all customers",
"instantiation_dict": {},
"intent": "Show all customers",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/customer/index/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 255
},
{
"sites": [
"shopping"
],
"task_id": 158,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 11
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 11 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 159,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 31
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 31 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 160,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 6
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 6 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 161,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 23
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 23 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 162,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
"instantiation_dict": {
"num": 40
},
"intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 40 cards",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 171
},
{
"sites": [
"shopping"
],
"task_id": 163,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"I ordered the 16gb but I only got 14 gigs even though I formatted the card",
"The memory card is kind of slow on games and downloads",
"No original packaging It's used and the previous owners data has not been erased",
"The product is a legit sony hardware that have been owned by someone else before",
"The media could not be loaded",
"I could not format the card so I wasn\u2019t able to use it for my VITA"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "I ordered the 16gb but I only got 14 gigs even though I formatted the card. The memory card is kind of slow on games and downloads. No original packaging It's used and the previous owners data has not been erased. The product is a legit sony hardware that have been owned by someone else before The media could not be loaded. I could not format the card so I wasn\u2019t able to use it for my VITA"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 164,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/mineralogie-all-natural-lip-gloss-ruby-rose.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Dry",
"Uneven color"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "DryUneven color"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 165,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/sandgrens-swedish-handmade-wooden-clog-sandal-copenhagen.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The 39 was too small. I am afraid the 40 will be too big",
"I was very sad when the shoe rubbed up against my baby toe",
"I had to return them because I knew in time it would tear up my feet",
"The problem is that the strap is made of some really stiff leather and is painful to my heel",
"The front is also uncomfortably tight",
"The Dansko's were similar (not as bad) and loosened up over time"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The 39 was too small. I am afraid the 40 will be too big. I was very sad when the shoe rubbed up against my baby toe. I had to return them because I knew in time it would tear up my feet. The problem is that the strap is made of some really stiff leather and is painful to my heel. The front is also uncomfortably tight. The Dansko's were similar (not as bad) and loosened up over time."
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 166,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/sensodyne-repair-protect-whitening-toothpaste-with-fluoride-3-4-oz-pack-of-3.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "there is no existing criticism",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 136
},
{
"sites": [
"shopping"
],
"task_id": 167,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/photosmart-plus-b209-clr-inkjetfb-p-s-c-usb-wrls-1.html",
"geolocation": null,
"intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
"instantiation_dict": {},
"intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"The wireless connection works on a whim (about 40% of the time I've owned it)",
"It seems to constantly run out of ink",
"Cartridge prices are less than some printers I've had",
"This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason",
"Scanner is as slow as my first scanner I ever owned in the mid-90's",
"For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "The wireless connection works on a whim (about 40% of the time I've owned it). It seems to constantly run out of ink. Cartridge prices are less than some printers I've had, but now I understand why. This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason. Scanner is as slow as my first scanner I ever owned in the mid-90's. For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
},
"intent_template_id": 136
},
{
"sites": [
"gitlab"
],
"task_id": 168,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "more than 100"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got more than 100 stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "No repo found",
"reference_answer_raw_annotation": "No repo found"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 169,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "the most"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got the most stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"a11yproject.com",
"design"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "a11yproject.com, Primer/design"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 170,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "the least"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got the least stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 171,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "less than 5"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got less than 5 stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"a11y-syntax-highlighting",
"a11y-webring.club",
"accessible-html-content-patterns",
"ericwbailey.website",
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "a11y-syntax-highlighting, a11y-webring.club, accessible-html-content-patterns, ericwbailey.website, cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 172,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
"instantiation_dict": {
"description": "no"
},
"intent": "Tell me the full names of the repositories where I made contributions and they got no stars?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"cloud-to-butt",
"dotfiles",
"timeit",
"solarized-prism-theme",
"gimmiethat.space",
"remove-board-movement-events-from-the-github-issue-timeline"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
},
"intent_template_id": 289
},
{
"sites": [
"gitlab"
],
"task_id": 173,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "better"
},
"intent": "Open my latest updated issue that has keyword \"better\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/8",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": "",
"url_note": "GOLD in PRED"
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 174,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "feature"
},
"intent": "Open my latest updated issue that has keyword \"feature\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-webring.club/-/issues/71",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 175,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "dependency"
},
"intent": "Open my latest updated issue that has keyword \"dependency\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 176,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "theme editor"
},
"intent": "Open my latest updated issue that has keyword \"theme editor\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 177,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
"instantiation_dict": {
"keyword": "homepage content"
},
"intent": "Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
"program_html": [],
"reference_answer_raw_annotation": "closed",
"string_note": ""
},
"intent_template_id": 310
},
{
"sites": [
"gitlab"
],
"task_id": 178,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "better"
},
"intent": "Open my latest created issue that has better in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/566",
"program_html": [],
"reference_answer_raw_annotation": "Closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 179,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "feature"
},
"intent": "Open my latest created issue that has feature in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1517",
"program_html": [],
"reference_answer_raw_annotation": "Closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 180,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "dependency"
},
"intent": "Open my latest created issue that has dependency in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 181,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "theme editor"
},
"intent": "Open my latest created issue that has theme editor in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["No, it is open"]
},
"reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
"program_html": [],
"reference_answer_raw_annotation": "Not closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"gitlab"
],
"task_id": 182,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
"instantiation_dict": {
"keyword": "homepage content"
},
"intent": "Open my latest created issue that has homepage content in its title to check if it is closed",
"require_reset": false,
"eval": {
"eval_types": [
"string_match",
"url_match"
],
"reference_answers": {
"fuzzy_match": ["Yes, it is closed"]
},
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
"program_html": [],
"reference_answer_raw_annotation": "closed",
"string_note": ""
},
"intent_template_id": 500
},
{
"sites": [
"shopping_admin"
],
"task_id": 183,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "SKU",
"N": "10"
},
"intent": "Give me the SKU of the products that have 10 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no product that has 10 quantities left.",
"reference_answer_raw_annotation": "There is no product that has 10 quantities left."
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 184,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "name",
"N": "0"
},
"intent": "Give me the name of the products that have 0 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Sinbad Fitness Tank"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sinbad Fitness Tank"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 185,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "brand",
"N": "3"
},
"intent": "Give me the brand of the products that have 3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Eos",
"Minerva"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Eos, Minerva"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 186,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "product names and the sizes",
"N": "2-3"
},
"intent": "Give me the product names and the sizes of the products that have 2-3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Eos V-Neck Hoodie: S",
"Minera Luma Tech V-Tee: XS"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Eos V-Neck Hoodie: S Minera Luma Tech V-Tee: XS"
},
"intent_template_id": 368
},
{
"sites": [
"shopping_admin"
],
"task_id": 187,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
"instantiation_dict": {
"Attribute": "SKU",
"N": "1-3"
},
"intent": "Give me the SKU of the products that have 1-3 units left",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"WH11-S-Blue",
"WS08-XS-Blue"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "WH11-S-Blue, WS08-XS-Blue"
},
"intent_template_id": 368
},
{
"sites": [
"shopping"
],
"task_id": 188,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Tell me the total cost of my latest cancelled order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"365.42"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "365.42"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 189,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "pending"
},
"intent": "Tell me the total cost of my latest pending order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"754.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "754.99"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 190,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "complete"
},
"intent": "Tell me the total cost of my latest complete order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"65.32"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "65.32"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 191,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "processing"
},
"intent": "Tell me the total cost of my latest processing order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no order of \"processing\" status",
"reference_answer_raw_annotation": "There is no order of \"processing\" status"
},
"intent_template_id": 214
},
{
"sites": [
"shopping"
],
"task_id": 192,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me the total cost of my latest {{status}} order?",
"instantiation_dict": {
"status": "non-cancelled"
},
"intent": "Tell me the total cost of my latest non-cancelled order?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"754.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "754.99"
},
"intent_template_id": 214
},
{
"sites": [
"shopping_admin"
],
"task_id": 193,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "completed",
"N": "2"
},
"intent": "Get the total payment amount of the last 2 completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"182.4"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "182.4"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 194,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "completed",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"555.2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "555.2"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 195,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "pending",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 pending orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"885.4"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "885.4"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 196,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Compare the payment difference of the last {{N}} {{status_1}} orders and {{status_2}} orders",
"instantiation_dict": {
"status_1": "cancelled",
"status_2": "completed",
"N": "4"
},
"intent": "Compare the payment difference of the last 4 cancelled orders and completed orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"194.25"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "194.25"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 197,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
"instantiation_dict": {
"status": "non-cancelled",
"N": "5"
},
"intent": "Get the total payment amount of the last 5 non-cancelled orders",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"778.2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"annotation_note": "219.4+210+166.4+93.4+89",
"reference_answer_raw_annotation": "778.2"
},
"intent_template_id": 367
},
{
"sites": [
"shopping_admin"
],
"task_id": 198,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "customer name",
"status": "most recent cancelled"
},
"intent": "Get the customer name of the most recent cancelled order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Lily Potter"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lily Potter"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 199,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "order ID",
"status": "newest pending"
},
"intent": "Get the order ID of the newest pending order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"299"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "299"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 200,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "billing name",
"status": "oldest complete"
},
"intent": "Get the billing name of the oldest complete order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "John Lee"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "John Lee"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 201,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "customer name",
"status": "earliest fraud suspect"
},
"intent": "Get the customer name of the earliest fraud suspect order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no order of \"fraud suspect\" status",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 202,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "date",
"status": "most recent canlled"
},
"intent": "Get the date of the most recent canlled order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"May 23 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "May 23, 2023"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 203,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "purchase date and order id",
"status": "most recent pending"
},
"intent": "Get the purchase date and order id of the most recent pending order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"order id: 000000299",
"purchase date: May 31, 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000299, May 31, 2023, 2:55:09 AM"
},
"intent_template_id": 366
},
{
"sites": [
"shopping_admin"
],
"task_id": 204,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Get the {{attribute}} of the {{status}} order",
"instantiation_dict": {
"attribute": "product name and discounted price (low to high)",
"status": "most recent completed"
},
"intent": "Get the product name and discounted price (low to high) of the most recent completed order",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Rapha Sports Short: $35",
"Thorpe Track Pant: $54.4",
"Mach Street Sweatshirt: $62"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Rapha Sports Short: $35 Thorpe Track Pant: $54.4 Mach Street Sweatshirt: $62"
},
"intent_template_id": 366
},
{
"sites": [
"gitlab"
],
"task_id": 205,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}}?",
"instantiation_dict": {
"user": "kilian",
"date": "3/5/2023"
},
"intent": "How many commits did kilian make on 3/5/2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 320
},
{
"sites": [
"gitlab"
],
"task_id": 206,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}}?",
"instantiation_dict": {
"user": "Eric",
"date": "3/2"
},
"intent": "How many commits did Eric make on 3/2?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"2"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2"
},
"intent_template_id": 320
},
{
"sites": [
"gitlab"
],
"task_id": 207,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make on {{date}} in total?",
"instantiation_dict": {
"user": "Eric and Kilian",
"date": "1/3/2023"
},
"intent": "How many commits did Eric and Kilian make on 1/3/2023 in total?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 320
},
{
"sites": [
"shopping_admin"
],
"task_id": 208,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "+1 2058812302"
},
"intent": "Find the customer name and email with phone number +1 2058812302",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"John Smith",
"john.smith.xyz@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "John Smith, john.smith.xyz@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 209,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "2137418080"
},
"intent": "Find the customer name and email with phone number 2137418080",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Jennifer White",
"jennifer.white@yahoo.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jennifer White, jennifer.white@yahoo.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 210,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "2065555555"
},
"intent": "Find the customer name and email with phone number 2065555555",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Adam Garcia",
"gamingpro456@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Adam Garcia, gamingpro456@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 211,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "8015551212"
},
"intent": "Find the customer name and email with phone number 8015551212",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sean Miller",
"sean.miller@gmail.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sean Miller, sean.miller@gmail.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 212,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
"instantiation_dict": {
"PhoneNum": "555-229-3326"
},
"intent": "Find the customer name and email with phone number 555-229-3326",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Veronica Costello",
"roni_cost@example.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Veronica Costello, roni_cost@example.com"
},
"intent_template_id": 364
},
{
"sites": [
"shopping_admin"
],
"task_id": 213,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "What are the key aspects that the customers don't like about Antonia Racer Tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Not suitable for high-impact workouts"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Not suitable for high-impact workouts"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 214,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Zing Jump Rope"
},
"intent": "What are the key aspects that the customers don't like about Zing Jump Rope",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"It is hard to find the right size. Won't last long"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "It is hard to find the right size. Won't last long"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 215,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Circe ice fleece"
},
"intent": "What are the key aspects that the customers don't like about Circe ice fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Material quality, fit, insufficient warmth, color"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Material quality, fit, insufficient warmth, color"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 216,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Electra Bra Top"
},
"intent": "What are the key aspects that the customers don't like about Electra Bra Top",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Not true to size"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Not true to size"
},
"intent_template_id": 249
},
{
"sites": [
"shopping_admin"
],
"task_id": 217,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "What are the key aspects that the customers don't like about {{product}}",
"instantiation_dict": {
"product": "Pursuit Tone Band"
},
"intent": "What are the key aspects that the customers don't like about Pursuit Tone Band",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Insufficient resistance for their workouts."
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Insufficient resistance for their workouts."
},
"intent_template_id": 249
},
{
"sites": [
"map"
],
"task_id": 218,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "CMU, Pittsburgh",
"n": "5"
},
"intent": "Show me the walking distance from nearby hotels to CMU, Pittsburgh that take at most 5 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 219,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "Pittsburgh airport",
"n": "3"
},
"intent": "Show me the walking distance from nearby hotels to Pittsburgh airport that take at most 3 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 220,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
"instantiation_dict": {
"location": "Gardner Steel Conference Center,",
"n": 5
},
"intent": "Show me the walking distance from nearby hotels to Gardner Steel Conference Center, that take at most 5 minutes?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Wyndham Pittsburgh University Cente: 375m",
"The Oaklander Hotel: 338m"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Wyndham Pittsburgh University Cente: 375 m\nThe Oaklander Hotel: 338 m"
},
"intent_template_id": 41
},
{
"sites": [
"map"
],
"task_id": 221,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to the nearest {{location}} with different transportation methods?",
"instantiation_dict": {
"location": "USPS postal office"
},
"intent": "I am at CMU Pittsburgh, how long it takes to the nearest USPS postal office with different transportation methods?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Walk: 1 minute",
"Drive: less than 1 minute",
"Bike: less than 1 minute"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Walk: 1 minute to walk and\nDrive: less than 1 minute\nBike: less than 1 minute"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 222,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "cold stone ice cream"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest cold stone ice cream",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3min"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 223,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "Mcdonald's"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest Mcdonald's",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"4min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4min"
},
"intent_template_id": 35
},
{
"sites": [
"map"
],
"task_id": 224,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
"instantiation_dict": {
"location": "wendys"
},
"intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest wendys",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"3min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3min"
},
"intent_template_id": 35
},
{
"sites": [
"shopping"
],
"task_id": 225,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What do customers say about {{product_type}} from {{manufature}}",
"instantiation_dict": {
"product_type": "brush",
"manufature": "sephora"
},
"intent": "What do customers say about brush from sephora",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "The sephora brushes don't have reviews",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 135
},
{
"sites": [
"shopping"
],
"task_id": 226,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "Amazon basic"
},
"intent": "What is the price range for products from Amazon basic?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"5.49",
"375.19"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$5.49 - $375.19"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 227,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "EYZUTAK"
},
"intent": "What is the price range for products from EYZUTAK?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$9.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 228,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "sephora"
},
"intent": "What is the price range for products from sephora?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"18.18",
"94.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$18.18 - $94.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 229,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "ugreen"
},
"intent": "What is the price range for products from ugreen?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"6.99",
"38.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$6.99 - $38.99"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 230,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the price range for products from {{brand}}?",
"instantiation_dict": {
"brand": "Perricone MD"
},
"intent": "What is the price range for products from Perricone MD?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"35",
"149"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$35 - $149"
},
"intent_template_id": 370
},
{
"sites": [
"shopping"
],
"task_id": 231,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Get the order number of my most recent cancelled order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"170"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000170"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 232,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "pending"
},
"intent": "Get the order number of my most recent pending order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"189"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000189"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 233,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "complete"
},
"intent": "Get the order number of my most recent complete order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"180"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "000000180"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 234,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "on hold"
},
"intent": "Get the order number of my most recent on hold order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "there is no on hold order",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 213
},
{
"sites": [
"shopping"
],
"task_id": 235,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Get the order number of my most recent {{status}} order ",
"instantiation_dict": {
"status": "under delivery"
},
"intent": "Get the order number of my most recent under delivery order ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no under delivery order",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 213
},
{
"sites": [
"map"
],
"task_id": 236,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
"instantiation_dict": {
"location": "pharmacy",
"location2": "Carnegie Mellon",
"condition": "I can walk within 20mins"
},
"intent": "Where is the nearest pharmacy from Carnegie Mellon I can walk within 20mins",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Schiller's Pharmacy",
"811",
"South Aiken Avenue",
"Shadyside",
"Pittsburgh"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Schiller's Pharmacy, 811, South Aiken Avenue, Shadyside, Pittsburgh, Allegheny County, 15232, United States"
},
"intent_template_id": 39
},
{
"sites": [
"map"
],
"task_id": 237,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
"instantiation_dict": {
"location": "gas station",
"location2": "CMU",
"condition": ""
},
"intent": "Where is the nearest gas station from CMU ",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Sunoco",
"North Craig Street",
"North Oakland",
"Pittsburgh"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Sunoco, North Craig Street, North Oakland, Pittsburgh, Allegheny County, 15213, United States"
},
"intent_template_id": 39
},
{
"sites": [
"shopping"
],
"task_id": 238,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "PS4 accessories"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from PS4 accessories category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/astro-gaming-a50-wireless-headset-base-station-gen-4-compatible-with-ps5-ps4-pc-mac-black-silver.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 239,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "nutrition bars and drinks"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from nutrition bars and drinks category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/kellogg-s-special-k-protein-meal-bars-chocolate-caramel-12-7oz-6-count.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 240,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "competitive swimwear"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from competitive swimwear category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/women-cross-flower-beachwear-tankini-bandeau-bandage-bikini-set-push-up-swimwear-bathing-suit-two-pieces-swimsuits.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 241,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "skin care tool"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from skin care tool category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/professional-medi-spa-scar-stretch-mark-reduction-system.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping"
],
"task_id": 242,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
"instantiation_dict": {
"product_category": "Household Supplies"
},
"intent": "I am doing a market survey for one stop market, show me the most expensive product from Household Supplies category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/lynx-battery-12v-200ah-lithium-iron-phosphate-lifepo4-prismatic-deep-cell-battery-set-of-4-3-2v-cells-with-3-bus-bars-and-8-lug-nuts-for-rv-solar-marine-off-grid-applications.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 138
},
{
"sites": [
"shopping_admin"
],
"task_id": 243,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "Circe fleece"
},
"intent": "Show me the email address of the customer who is the most unhappy with Circe fleece",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "hannah.lim@gmail.com"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "hannah.lim@gmail.com"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 244,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "Olivia zip jacket"
},
"intent": "Show me the email address of the customer who is the most unhappy with Olivia zip jacket",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "emma.lopez@gmail.com"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "emma.lopez@gmail.com"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 245,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "name",
"product": "Antonia racer tank"
},
"intent": "Show me the name of the customer who is the most unhappy with Antonia racer tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Shaunte"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shaunte"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 246,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "name",
"product": "Chloe tank"
},
"intent": "Show me the name of the customer who is the most unhappy with Chloe tank",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Teofila"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Teofila"
},
"intent_template_id": 244
},
{
"sites": [
"shopping_admin"
],
"task_id": 247,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
"instantiation_dict": {
"information": "email address",
"product": "the style of Zoe products"
},
"intent": "Show me the email address of the customer who is the most unhappy with the style of Zoe products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "Valorie doesn't have a email in the system",
"program_html": [],
"string_note": "There is no negative review for Zoe products",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 244
},
{
"sites": [
"map"
],
"task_id": 248,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Carnegie Mellon Caf\u00e9"
},
"intent": "Tell me the coordinates of Carnegie Mellon Caf\u00e9 in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.442",
"-79.939"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4424191, -79.9397388"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 249,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Western Pennsylvania Hospital Heliport"
},
"intent": "Tell me the coordinates of Western Pennsylvania Hospital Heliport in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.460",
"-79.946"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.46076, -79.94666"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 250,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Apple Store near Pitt"
},
"intent": "Tell me the coordinates of Apple Store near Pitt in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.451",
"-79.933"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4511693, -79.9334241"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 251,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "bus stop on the Carnegie art museum side of the street near CMU"
},
"intent": "Tell me the coordinates of bus stop on the Carnegie art museum side of the street near CMU in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.444",
"-79.948"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.4443, -79.94889"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 252,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Tell me the coordinates of {{location}} in DD format",
"instantiation_dict": {
"location": "Tokyo Japanese Food Store in Pittsburgh"
},
"intent": "Tell me the coordinates of Tokyo Japanese Food Store in Pittsburgh in DD format",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.457",
"-79.929"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.45761, -79.92934"
},
"intent_template_id": 46
},
{
"sites": [
"map"
],
"task_id": 253,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Carnegie Mellon Caf\u00e9",
"information": "phone number"
},
"intent": "What is the phone number of Carnegie Mellon Caf\u00e9",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no such information in the map",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 254,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Western Pennsylvania Hospital",
"information": "phone number"
},
"intent": "What is the phone number of Western Pennsylvania Hospital",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": ["4125785000"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "4125785000"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 255,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Who is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "PIT airport",
"information": "operator"
},
"intent": "Who is the operator of PIT airport",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Allegheny County Airport Authority"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Allegheny County Airport Authority"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 256,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Carnegie art museum in pittsburgh",
"information": "website"
},
"intent": "What is the website of Carnegie art museum in pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "http://web.cmoa.org/"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "http://web.cmoa.org/"
},
"intent_template_id": 501
},
{
"sites": [
"map"
],
"task_id": 257,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What is the {{information}} of {{location}}",
"instantiation_dict": {
"location": "Tokyo Japanese Food Store in Pittsburgh",
"information": "hours of operation"
},
"intent": "What is the hours of operation of Tokyo Japanese Food Store in Pittsburgh",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"We-Su 10:00-17:00"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "We-Su 10:00-17:00"
},
"intent_template_id": 501
},
{
"sites": [
"gitlab"
],
"task_id": 258,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "See all public projects",
"instantiation_dict": {},
"intent": "See all public projects",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/explore",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 325
},
{
"sites": [
"gitlab"
],
"task_id": 259,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Get me my RSS feed token",
"instantiation_dict": {},
"intent": "Get me my RSS feed token",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "TMN_bBn9Z48qVbUFZV45"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "TMN_bBn9Z48qVbUFZV45"
},
"intent_template_id": 312
},
{
"sites": [
"shopping"
],
"task_id": 260,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Video Game"
},
"intent": "I want to browse the products in the Video Game category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/video-games.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 261,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Headphones"
},
"intent": "I want to browse the products in the Headphones category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/electronics/headphones.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/electronics.html?cat=60"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 262,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Men shoes"
},
"intent": "I want to browse the products in the Men shoes category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=145"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 263,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Woman clothing"
},
"intent": "I want to browse the products in the Woman clothing category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/clothing.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=143"}]
},
"intent_template_id": 211
},
{
"sites": [
"shopping"
],
"task_id": 264,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I want to browse the products in the {{category}} category",
"instantiation_dict": {
"category": "Cabinets, Racks & Shelves"
},
"intent": "I want to browse the products in the Cabinets, Racks & Shelves category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/office-products/office-furniture-lighting/cabinets-racks-shelves.html",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/office-products.html?cat=187"}]
},
"intent_template_id": 211
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 265,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How far is it to drive there?",
"instantiation_dict": {
"city": "Boston"
},
"intent": "What's the closest national park to Boston? How far is it to drive there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park",
"457km"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n457km"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 266,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}?",
"instantiation_dict": {
"city": "the largest city in Maine"
},
"intent": "What's the closest national park to the largest city in Maine?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Acadia National Park"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 267,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How long it takes to drive there?",
"instantiation_dict": {
"city": "the hometown of Stephen King"
},
"intent": "What's the closest national park to the hometown of Stephen King? How long it takes to drive there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park"
],
"fuzzy_match": [
"1h 23min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n1h 23min"
},
"intent_template_id": 85
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 268,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "What's the closest national park to {{city}}? How long does it take to bike there?",
"instantiation_dict": {
"city": "Vinalhaven, ME"
},
"intent": "What's the closest national park to Vinalhaven, ME? How long does it take to bike there?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Acadia National Park"
],
"fuzzy_match": [
"10h 33min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Acadia National Park\n10h 33min"
},
"intent_template_id": 85
},
{
"sites": [
"shopping"
],
"task_id": 269,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "25",
"product_category": "women shoes"
},
"intent": "Show me products under $25 in \"women shoes\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/shoes.html?price=0-25",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 270,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "30",
"product_category": "men shoes"
},
"intent": "Show me products under $30 in \"men shoes\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html?price=0-30",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 271,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "46.99",
"product_category": "makeup remover"
},
"intent": "Show me products under $46.99 in \"makeup remover\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/beauty-personal-care/makeup/makeup-remover.html?price=0-46.99",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 272,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "78",
"product_category": "children dental care"
},
"intent": "Show me products under $78 in \"children dental care\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/beauty-personal-care/oral-care/children-s-dental-care.html?price=0-78",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 273,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
"instantiation_dict": {
"price": "199",
"product_category": "furtiture with accent"
},
"intent": "Show me products under $199 in \"furtiture with accent\" category",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/furniture/accent-furniture.html?price=0-199",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 139
},
{
"sites": [
"shopping"
],
"task_id": 274,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "usb wifi"
},
"intent": "Search for \"usb wifi\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=usb+wifi",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 275,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "xbox"
},
"intent": "Search for \"xbox\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=xbox",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 276,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "switch accessories"
},
"intent": "Search for \"switch accessories\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=switch+accessories",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 277,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "batteries for iphone 13"
},
"intent": "Search for \"batteries for iphone 13\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=iphone+13",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 278,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Search for \"{{keyword}}\"",
"instantiation_dict": {
"keyword": "green tea bag for weight loss"
},
"intent": "Search for \"green tea bag for weight loss\"",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=green+tea+bag+for+weight+loss",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 212
},
{
"sites": [
"shopping"
],
"task_id": 279,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
"instantiation_dict": {},
"intent": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed)",
"Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones",
"Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus",
"Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black",
"Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items)",
"Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W)",
"Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B)",
"Sony MDRAS600BT Active Sports Bluetooth Headset (Black)",
"Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items)",
"Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)",
"Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black",
"SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed",
"18.99",
"406"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are avaiable: SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed) Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items) Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W) Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B) Sony MDRAS600BT Active Sports Bluetooth Headset (Black) Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items) Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R) Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed The price ranges from $18.99 to $406 "
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 280,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
"instantiation_dict": {},
"intent": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More",
"Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)",
"Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)",
"5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply",
"Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)",
"Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)",
"USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More",
"iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy",
"USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More",
"Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)",
"Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More",
"USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)",
"8.99",
"59.99"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included) Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278) 5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter) Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter) USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111) Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included) Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built-in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter) USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included) The price ranges from $8.99 to $59.99"
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 281,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
"instantiation_dict": {},
"intent": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count",
"Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack",
"3.745",
"6.495"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack The price ranges from $3.745 to $6.495 "
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 282,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
"instantiation_dict": {},
"intent": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Nike Men's Air Max Camden Slide Sandal",
"Nike Men's Benassi JDI Fanny Pack Slides",
"Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)",
"Nike Offcourt Slide Mens Bq4639-002 Size 12",
"Nike Jordan Men's Break Slide Red AR6374-602",
"Nike Victori One Slide Mens Style : Dd9559-300",
"Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)",
"Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)",
"Nike womens Benassi Just Do It",
"27.6",
"90.65"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "These models are availiable: Nike Men's Air Max Camden Slide Sandal Nike Men's Benassi JDI Fanny Pack Slides Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10) Nike Offcourt Slide Mens Bq4639-002 Size 12 Nike Jordan Men's Break Slide Red AR6374-602 Nike Victori One Slide Mens Style : Dd9559-300 Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14) Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8) Nike womens Benassi Just Do It The price ranges from $27.6 to $90.65"
},
"intent_template_id": 204
},
{
"sites": [
"shopping"
],
"task_id": 283,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Look up the most recent models of XBox controllers released between 2020-2021?",
"instantiation_dict": {},
"intent": "Look up the most recent models of XBox controllers released between 2020-2021?",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/microsoft-xbox-controller-carbon-black-for-series-x-series-s-xbox-one-windows-10-android-ios-bundled-with-dual-port-charging-dock-xbox-controller-skin-voucher-premgear-cloth.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 210
},
{
"sites": [
"shopping"
],
"task_id": 284,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "shoe storage",
"min_storage": "12 pairs"
},
"intent": "Show the least expensive shoe storage with a minimum storage capacity of 12 pairs.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/onlyeasy-over-the-door-shoe-storage-organizer-hanging-shoe-rack-holder-with-24-large-fabric-pockets-22-1-x-61-4-herringbone-grey-mxrodsb1p.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"shopping"
],
"task_id": 285,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "switch card holder",
"min_storage": "15 cards"
},
"intent": "Show the least expensive switch card holder with a minimum storage capacity of 15 cards.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"shopping"
],
"task_id": 286,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
"instantiation_dict": {
"product": "ssd hard drive",
"min_storage": "1TB"
},
"intent": "Show the least expensive ssd hard drive with a minimum storage capacity of 1TB.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/external-hard-drive-2tb-ultra-thin-external-hard-drive-2000gb-ultra-high-speed-portable-3-1-type-c-storage-drive-compatible-with-pc-laptop-and-mac-2tb-a1.html",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 207
},
{
"sites": [
"map"
],
"task_id": 287,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "How much time does it take from Pittsburgh to Philadelphia by car?",
"instantiation_dict": {},
"intent": "How much time does it take from Pittsburgh to Philadelphia by car?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"5h 47min"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "5h 47min"
},
"intent_template_id": 47
},
{
"sites": [
"shopping_admin"
],
"task_id": 288,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "name"
},
"intent": "Tell me the name of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "Samantha Jones"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Samantha Jones"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 289,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "email address, name, phone number"
},
"intent": "Tell me the email address, name, phone number of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"email: coolcat321@hotmail.com",
"name: Samantha Jones",
"phone number: 3055551212"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "email: coolcat321@hotmail.com name: Samantha Jones phone number: 3055551212"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 290,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "product SKUs in the most recent cancelled orders"
},
"intent": "Tell me the product SKUs in the most recent cancelled orders of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"WSH09-29-White",
"WSH09-28-Green",
"MSH11-34-Blue",
"WP09-29-Purple"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "WSH09-29-White,WSH09-28-Green,MSH11-34-Blue,WP09-29-Purple"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 291,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "total spend on products in the most recent cancelled orders"
},
"intent": "Tell me the total spend on products in the most recent cancelled orders of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"148"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "$148"
},
"intent_template_id": 234
},
{
"sites": [
"shopping_admin"
],
"task_id": 292,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
"instantiation_dict": {
"attribute": "total number of cancellations"
},
"intent": "Tell me the total number of cancellations of the customer who has the most cancellations in the history",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"9"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "9"
},
"intent_template_id": 234
},
{
"sites": [
"gitlab"
],
"task_id": 293,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "Super_Awesome_Robot"
},
"intent": "Show me the command to clone Super_Awesome_Robot with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 294,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "ChatGPT"
},
"intent": "Show me the command to clone ChatGPT with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 295,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "metaseq"
},
"intent": "Show me the command to clone metaseq with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 296,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "the best GAN python implementation"
},
"intent": "Show me the command to clone the best GAN python implementation with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"
},
"intent_template_id": 329
},
{
"sites": [
"gitlab"
],
"task_id": 297,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Show me the command to clone {{repo}} with SSH.",
"instantiation_dict": {
"repo": "the most stared Covid location tracker"
},
"intent": "Show me the command to clone the most stared Covid location tracker with SSH.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"
},
"intent_template_id": 329
},
{
"sites": [
"shopping"
],
"task_id": 298,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "completed"
},
"intent": "Show the most recent completed order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/180/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 299,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "cancelled"
},
"intent": "Show the most recent cancelled order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/170/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 300,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "pending"
},
"intent": "Show the most recent pending order page",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/sales/order/view/order_id/189/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 301,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "processing"
},
"intent": "Show the most recent processing order page",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {"fuzzy_match": "N/A"},
"reference_url": "",
"program_html": [],
"string_note": "there is no order in processing"
},
"intent_template_id": 180
},
{
"sites": [
"shopping"
],
"task_id": 302,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show the most recent {{status}} order page",
"instantiation_dict": {
"status": "out of delivery"
},
"intent": "Show the most recent out of delivery order page",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {"fuzzy_match": "N/A"},
"reference_url": "",
"program_html": [],
"string_note": "there is no order in processing"
},
"intent_template_id": 180
},
{
"sites": [
"gitlab"
],
"task_id": 303,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Kilian",
"period": "durning 2023"
},
"intent": "How many commits did Kilian make durning 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"1"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 304,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Eric",
"period": "between Feb 2023 and May 2023"
},
"intent": "How many commits did Eric make between Feb 2023 and May 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"14"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "14"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 305,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Philip",
"period": "in 2023/1"
},
"intent": "How many commits did Philip make in 2023/1?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 306,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Anthony",
"period": "between 08/2022-09/2022"
},
"intent": "How many commits did Anthony make between 08/2022-09/2022?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 307,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "How many commits did {{user}} make {{period}}?",
"instantiation_dict": {
"user": "Nic",
"period": "in April 2021"
},
"intent": "How many commits did Nic make in April 2021?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "16"
},
"intent_template_id": 321
},
{
"sites": [
"gitlab"
],
"task_id": 308,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "primer/design"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the primer/design project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Shawn Allen"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shawn Allen"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 309,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "thoughtbot/administrate"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the thoughtbot/administrate project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Grayson Wright"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Grayson Wright"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 310,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "AndroidSlidingUpPanel"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the AndroidSlidingUpPanel project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["tokudu"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "tokudu"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 311,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "Pytorch GAN"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the Pytorch GAN project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Erik Linder-Nor\u00e9n"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Erik Linder-Nor\u00e9n"
},
"intent_template_id": 323
},
{
"sites": [
"gitlab"
],
"task_id": 312,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
"instantiation_dict": {
"repo": "csvkit"
},
"intent": "Tell me who has made the most contributions, in terms of number of commits, to the csvkit project",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["Christopher Groskopf"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Christopher Groskopf"
},
"intent_template_id": 323
},
{
"sites": [
"shopping"
],
"task_id": 313,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Which number to call for the customer service?",
"instantiation_dict": {},
"intent": "Which number to call for the customer service?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no phone number in the website",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 134
},
{
"sites": [
"gitlab"
],
"task_id": 314,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "prime/design",
"attribute": "name"
},
"intent": "List the name of the top 3 contributors to prime/design repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Shawn Allen",
"Inayaili Le\u00f3n",
"Aurora Pleguezuelo"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Shawn Allen, Inayaili Le\u00f3n, Aurora Pleguezuelo"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 315,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "Pytorch GAN",
"attribute": "email address"
},
"intent": "List the email address of the top 3 contributors to Pytorch GAN repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"eriklindernoren@live.se",
"eriklindernoren@gmail.com",
"pinnacle.chen@qq.com"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "eriklindernoren@live.se, eriklindernoren@gmail.com, pinnacle.chen@qq.com"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 316,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "facebook's guide on building react apps",
"attribute": "name"
},
"intent": "List the name of the top 3 contributors to facebook's guide on building react apps repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Ian Sutherland",
"Joe Hadda",
"Dan Abramov"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Ian Sutherland, Joe Hadda, Dan Abramov"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 317,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "metaseq",
"attribute": "name and number of commits"
},
"intent": "List the name and number of commits of the top 3 contributors to metaseq repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Susan Zhang: 70",
"Stephen Roller: 51",
"Peter Albert: 12"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Susan Zhang: 70, Stephen Roller: 51, Peter Albert: 12"
},
"intent_template_id": 324
},
{
"sites": [
"gitlab"
],
"task_id": 318,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
"instantiation_dict": {
"repo": "2019-nCov",
"attribute": "last names"
},
"intent": "List the last names of the top 3 contributors to 2019-nCov repo, ranked by the number of commits?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lo",
"Chen",
"Chu"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lo, Chen, Chu"
},
"intent_template_id": 324
},
{
"sites": [
"shopping"
],
"task_id": 319,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "April 2022"
},
"intent": "How much refund I should expect from my order canlled in April 2022, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 320,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "Feb 2023"
},
"intent": "How much refund I should expect from my order canlled in Feb 2023, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"406.53"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "406.53"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 321,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
"instantiation_dict": {
"time": "2022"
},
"intent": "How much refund I should expect from my order canlled in 2022, including shipping fee",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"3053.97"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "3053.97"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 322,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}} if I cannot get the shipping fee refunded?",
"instantiation_dict": {
"time": "May 2023"
},
"intent": "How much refund I should expect from my order canlled in May 2023 if I cannot get the shipping fee refunded?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"350.42"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "350.42"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 323,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much refund I should expect from my order canlled in {{time}}? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
"instantiation_dict": {
"time": "2022/03"
},
"intent": "How much refund I should expect from my order canlled in 2022/03? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"264.49"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "264.49"
},
"intent_template_id": 160
},
{
"sites": [
"shopping"
],
"task_id": 324,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "chairs",
"sorting_order": "ascending price"
},
"intent": "Show me the \"chairs\" listings by ascending price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=chairs&product_list_dir=asc",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 325,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "mouth night guard",
"sorting_order": "descending price"
},
"intent": "Show me the \"mouth night guard\" listings by descending price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?q=mouth%20night%20guard%20&product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 326,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "Canon photo printer",
"sorting_order": "search relevance, from most to least"
},
"intent": "Show me the \"Canon photo printer\" listings by search relevance, from most to least.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/?q=Canon+photo+printer",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 327,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "iphone 12 phone case",
"sorting_order": "name alphabetically"
},
"intent": "Show me the \"iphone 12 phone case\" listings by name alphabetically.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?q=%20iphone%2012%20phone%20case&product_list_order=name",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 328,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
"instantiation_dict": {
"product": "iphone 12 phone case",
"sorting_order": "price"
},
"intent": "Show me the \"iphone 12 phone case\" listings by price.",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=%20iphone%2012%20phone%20case",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 208
},
{
"sites": [
"shopping"
],
"task_id": 329,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "on 4/19/2023"
},
"intent": "How much I spend on 4/19/2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 330,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "in March 2023"
},
"intent": "How much I spend in March 2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"83.31"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "81.31"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 331,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "in July 2022"
},
"intent": "How much I spend in July 2022 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"40.16"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "40.16"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 332,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
"instantiation_dict": {
"time": "each month from Jan to the end of March 2023"
},
"intent": "How much I spend each month from Jan to the end of March 2023 on shopping at One Stop Market?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"Jan: 572.8",
"Feb: 762.18",
"Mar: 83.31"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Jan: 572.8\nFeb: 762.18\nMar: 83.31"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 333,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "How much did I spend on shopping at One Stop Market {{time}}? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
"instantiation_dict": {
"time": "on November 2022"
},
"intent": "How much did I spend on shopping at One Stop Market on November 2022? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"359.546"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "359.546"
},
"intent_template_id": 147
},
{
"sites": [
"shopping"
],
"task_id": 334,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "muffin cornbread mix"
},
"intent": "Tell me when I last ordered my muffin cornbread mix?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"March 11th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "March 11th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 335,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "body butter"
},
"intent": "Tell me when I last ordered my body butter?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January 16th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January 16th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 336,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "conditioner"
},
"intent": "Tell me when I last ordered my conditioner?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"January 16th 2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "January 16th 2023"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 337,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "bread olive"
},
"intent": "Tell me when I last ordered my bread olive?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"December 12th 2022"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "December 12th 2022"
},
"intent_template_id": 169
},
{
"sites": [
"shopping"
],
"task_id": 338,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Tell me when I last ordered my {{description}}?",
"instantiation_dict": {
"description": "toothpaste"
},
"intent": "Tell me when I last ordered my toothpaste?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"December 4th 2022"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "December 4th 2022"
},
"intent_template_id": 169
},
{
"sites": [
"gitlab"
],
"task_id": 339,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/a11yproject/a11yproject.com",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that report bugs"
},
"intent": "List all opened issues that report bugs",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?label_name%5B%5D=bug",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 340,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/primer/design",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that report bugs"
},
"intent": "List all opened issues that report bugs",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/primer/design/-/issues/?label_name%5B%5D=type%3A%20bug%20%F0%9F%90%9E",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 341,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "requesting new features"
},
"intent": "List all opened issues requesting new features",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=enhancement",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 342,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that ask about OPT model related questions"
},
"intent": "List all opened issues that ask about OPT model related questions",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?search=OPT&label_name%5B%5D=question",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"gitlab"
],
"task_id": 343,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/root/metaseq",
"geolocation": null,
"intent_template": "List all opened issues {{description}}",
"instantiation_dict": {
"description": "that don't have any labels"
},
"intent": "List all opened issues that don't have any labels",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=None",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 299
},
{
"sites": [
"shopping_admin"
],
"task_id": 344,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "by far"
},
"intent": "How many reviews our shop received by far?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 345,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "in Apr 2023"
},
"intent": "How many reviews our shop received in Apr 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 346,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "during 2022"
},
"intent": "How many reviews our shop received during 2022?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 347,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "from the beginning of the shop"
},
"intent": "How many reviews our shop received from the beginning of the shop?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"351"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "351"
},
"intent_template_id": 248
},
{
"sites": [
"shopping_admin"
],
"task_id": 348,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "How many reviews our shop received {{time}}?",
"instantiation_dict": {
"time": "in May 2023"
},
"intent": "How many reviews our shop received in May 2023?",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"0"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "0"
},
"intent_template_id": 248
},
{
"sites": [
"gitlab"
],
"task_id": 349,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
"instantiation_dict": {
"repo": "gimmiethat.space"
},
"intent": "Who else have access to my repo gimmiethat.space, show me their usernames",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"exact_match": "yjlou"
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "yjlou"
},
"intent_template_id": 298
},
{
"sites": [
"gitlab"
],
"task_id": 350,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
"instantiation_dict": {
"repo": "prism-theme"
},
"intent": "Who else have access to my repo prism-theme, show me their usernames",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"abisubramanya27"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Abishek S, abisubramanya27"
},
"intent_template_id": 298
},
{
"sites": [
"shopping"
],
"task_id": 351,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "PS4 accessories",
"order": "ascending"
},
"intent": "List products from PS4 accessories category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/video-games/playstation-4/accessories.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/video-games.html?cat=67&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 352,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "nutrition bars and drinks",
"order": "ascending"
},
"intent": "List products from nutrition bars and drinks category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/health-household/diet-sports-nutrition/nutrition-bars-drinks.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/health-household.html?cat=192&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 353,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "competitive swimwear",
"order": "ascending"
},
"intent": "List products from competitive swimwear category by ascending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/clothing-shoes-jewelry/sport-specific-clothing/competitive-swimwear.html?product_list_order=price",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=149&product_list_order=price"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 354,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "living room furtniture",
"order": "descending"
},
"intent": "List products from living room furtniture category by descending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/furniture/living-room-furniture.html?product_list_order=price&product_list_dir=desc",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=154&product_list_order=price&product_list_dir=desc"}]
},
"intent_template_id": 137
},
{
"sites": [
"shopping"
],
"task_id": 355,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List products from {{product_category}} category by {{order}} price",
"instantiation_dict": {
"product_category": "kids' bedding",
"order": "descending"
},
"intent": "List products from kids' bedding category by descending price",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING__/home-kitchen/bedding/kids-bedding.html?product_list_dir=desc",
"program_html": [],
"url_note": "GOLD in PRED",
"or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=155&product_list_dir=desc"}]
},
"intent_template_id": 137
},
{
"sites": [
"map"
],
"task_id": 356,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
"instantiation_dict": {},
"intent": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
"required_contents": {
"exact_match": "1"
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_from\"').value",
"required_contents": {
"must_include": [
"Gates and Hillman Centers",
"Pittsburgh"
]
}
},
{
"url": "last",
"locator": "document.querySelector('[name=\"route_to\"').value",
"required_contents": {
"must_include": [
"Independence Hall",
"Philadelphia"
]
}
}
]
},
"intent_template_id": 49
},
{
"sites": [
"gitlab"
],
"task_id": 357,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Checkout merge requests requiring my review",
"instantiation_dict": {},
"intent": "Checkout merge requests requiring my review",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__GITLAB__/dashboard/merge_requests?reviewer_username=byteblaze",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 291
},
{
"sites": [
"shopping"
],
"task_id": 358,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "shipping method",
"order_number": 187
},
"intent": "Show me the shipping method for order number 187.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Flat Rate - Fixed"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Flat Rate - Fixed"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 359,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "order date",
"order_number": "148"
},
"intent": "Show me the order date for order number 148.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"1/29/2023"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1/29/2023"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 360,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "product names",
"order_number": "148"
},
"intent": "Show me the product names for order number 148.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress)",
"Russound 5B45W 4\" Indoor Outdoor Speakers White"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress), Russound 5B45W 4\" Indoor Outdoor Speakers White"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 361,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "order statuses",
"order_number": "170 and 189"
},
"intent": "Show me the order statuses for order number 170 and 189.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": [
"170: cancelled",
"189: pending"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "170: cancelled, 189: pending"
},
"intent_template_id": 206
},
{
"sites": [
"shopping"
],
"task_id": 362,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Show me the {{info}} for order number {{order_number}}.",
"instantiation_dict": {
"info": "billing address",
"order_number": "00178"
},
"intent": "Show me the billing address for order number 00178.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"101 S San Mateo Dr",
"San Mateo",
"California",
"94010",
"United States"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Emma Lopez, 101 S San Mateo Dr, San Mateo, California, 94010, United States"
},
"intent_template_id": 206
},
{
"sites": [
"map"
],
"task_id": 363,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "Carnegie Music Hall"
},
"intent": "Measure distance between Carnegie Mellon University and Carnegie Music Hall by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["778m |OR| 778 m"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "748m"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 364,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between Carnegie Mellon University and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.7km |OR| 1.7 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.7km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 365,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Music Hall",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between Carnegie Music Hall and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["2.2km |OR| 2.2 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "2.2km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 366,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "CVS (closet one)",
"location/address_2": "UPMC Shadyside"
},
"intent": "Measure distance between CVS (closet one) and UPMC Shadyside by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.2km |OR| 1.2 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.2km"
},
"intent_template_id": 58
},
{
"sites": [
"map"
],
"task_id": 367,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
"instantiation_dict": {
"location/address_1": "Carnegie Mellon University",
"location/address_2": "CVS (closet one)"
},
"intent": "Measure distance between Carnegie Mellon University and CVS (closet one) by walking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": ["1.4km |OR| 1.4 km"]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "1.4km"
},
"intent_template_id": 58
},
{
"sites": [
"shopping"
],
"task_id": 368,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "find discounted items.",
"instantiation_dict": {},
"intent": "find discounted items.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no function to show only discount items",
"reference_answer_raw_annotation": "There is no function to show only discount items."
},
"intent_template_id": 188
},
{
"sites": [
"map"
],
"task_id": 369,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Carnegie Music Hall"
},
"intent": "Pull up the description page of Carnegie Music Hall on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Music Hall"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 370,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Carnegie Mellon University"
},
"intent": "Pull up the description page of Carnegie Mellon University on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Mellon University"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 371,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Piada restaurant near Pitt"
},
"intent": "Pull up the description page of Piada restaurant near Pitt on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Piada Italian Street Food",
"Forbes Avenue"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 372,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "the Costco in Pittsburhg near a river"
},
"intent": "Pull up the description page of the Costco in Pittsburhg near a river on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Costco",
"Waterfront Drive West"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"map"
],
"task_id": 373,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Pull up the description page of {{location}} on Map",
"instantiation_dict": {
"location": "Whole Foods near Carnegie Mellon"
},
"intent": "Pull up the description page of Whole Foods near Carnegie Mellon on Map",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": null,
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Whole Foods",
"East Liberty"
]
}
}
]
},
"intent_template_id": 52
},
{
"sites": [
"shopping_admin"
],
"task_id": 374,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Preview the {{name}} theme for my shop",
"instantiation_dict": {
"name": "Magento Blank"
},
"intent": "Preview the Magento Blank theme for my shop",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/1",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 266
},
{
"sites": [
"shopping_admin"
],
"task_id": 375,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Preview the {{name}} theme for my shop",
"instantiation_dict": {
"name": "Magento Luma"
},
"intent": "Preview the Magento Luma theme for my shop",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/3/key/",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 266
},
{
"sites": [
"shopping"
],
"task_id": 376,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Summarize customer reviews for {{product}}.",
"instantiation_dict": {
"product": "Amazon Echo Dot 3rd generation"
},
"intent": "Summarize customer reviews for Amazon Echo Dot 3rd generation.",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no review for this product",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 182
},
{
"sites": [
"map"
],
"task_id": 377,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU ArtPark Lab",
"space": "resturants"
},
"intent": "Find the resturants around CMU ArtPark Lab",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=restaurants%20near%20CMU%20ArtPark%20Lab",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 378,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU main campus",
"space": "parking"
},
"intent": "Find the parking around CMU main campus",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=parking%20near%20carnegie%20mellon%20university",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 379,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "CMU main campus",
"space": "hotel"
},
"intent": "Find the hotel around CMU main campus",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=hotels%20near%20carnegie%20mellon%20university",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 380,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "Carnegie Music Hall",
"space": "bar"
},
"intent": "Find the bar around Carnegie Music Hall",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=bars%20near%20Carnegie%20Music%20Hall",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 381,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the {{space}} around {{location}}",
"instantiation_dict": {
"location": "Carnegie Music Hall",
"space": "hotel"
},
"intent": "Find the hotel around Carnegie Music Hall",
"require_reset": false,
"eval": {
"eval_types": [
"url_match"
],
"reference_answers": null,
"reference_url": "__MAP__/search?query=hotels%20near%20Carnegie%20Music%20Hall",
"program_html": [],
"url_note": "GOLD in PRED"
},
"intent_template_id": 59
},
{
"sites": [
"map"
],
"task_id": 382,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
"instantiation_dict": {},
"intent": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "There is no USCIS nearby",
"reference_answer_raw_annotation": "N/A"
},
"intent_template_id": 781
},
{
"sites": [
"map"
],
"task_id": 383,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
"instantiation_dict": {},
"intent": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Hyatt Regency Pittsburgh International Airport",
"Giant Eagle",
"ALDI"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Giant Eagle, ALDI"
},
"intent_template_id": 782
},
{
"sites": [
"shopping"
],
"task_id": 384,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the customer names who complain about the quality of EYZUTAK phone cases",
"instantiation_dict": {},
"intent": "List the customer names who complain about the quality of EYZUTAK phone cases",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Lee",
"Evelyn Kurver",
"Amanda",
"N Randall"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Lee, Evelyn Kurver, Amanda, N Randall"
},
"intent_template_id": 666
},
{
"sites": [
"shopping"
],
"task_id": 385,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "List the customer names who thinks EYZUTAK phone cases are of good looking",
"instantiation_dict": {},
"intent": "List the customer names who thinks EYZUTAK phone cases are of good looking",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Lisa Lee",
"MH",
"Misba009",
"Amanda",
"N Randall",
"Amazon Customer",
"Cally",
"Bethany Robertson"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Lisa Lee, MH, Misba009, Amanda, N Randall, Amazon Customer, Cally, Bethany Robertson"
},
"intent_template_id": 666
},
{
"sites": [
"shopping"
],
"task_id": 386,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "What is the rating of {{product}}",
"instantiation_dict": {
"product": "Ugreen lightning to 3.5mm cable"
},
"intent": "What is the rating of Ugreen lightning to 3.5mm cable. Please round to the nearest whole number",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"65 |OR| 3"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Around 3.1 stars"
},
"intent_template_id": 1355
},
{
"sites": [
"shopping"
],
"task_id": 387,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
"instantiation_dict": {
"stars": "4 or 5 stars"
},
"intent": "Who gave 4 or 5 stars for phone cases from EYZUTAK",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"MH",
"Misba009",
"Amanda",
"Amazon Customer",
"Cally",
"Bethany Robertson"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "MH, Misba009, Amanda, Amazon Customer, Cally, Bethany Robertson, Bethany Robertson"
},
"intent_template_id": 1356
},
{
"sites": [
"shopping"
],
"task_id": 388,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
"instantiation_dict": {
"stars": "1 or 2 stars"
},
"intent": "Who gave 1 or 2 stars for phone cases from EYZUTAK",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"must_include": [
"Evelyn Kurver",
"N Randall"
]
},
"reference_url": "",
"program_html": [],
"string_note": "",
"reference_answer_raw_annotation": "Evelyn Kurver, N Randall"
},
"intent_template_id": 1356
},
{
"sites": [
"gitlab"
],
"task_id": 389,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "octovisuals page",
"repo": "primer/design",
"content": "Thanks, working on reviews"
},
"intent": "Post \"Thanks, working on reviews\" for the merge request related to octovisuals page in primer/design project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/primer/design/-/merge_requests/450",
"locator": "",
"required_contents": {
"must_include": [
"Thanks, working on reviews"
]
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 390,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "semantic HTML post",
"repo": "a11yproject/a11yproject.com",
"content": "lgtm"
},
"intent": "Post \"lgtm\" for the merge request related to semantic HTML post in a11yproject/a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1531",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "lgtm"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 391,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "focus edge cases",
"repo": "a11yproject/a11yproject.com",
"content": "close because non reproducible"
},
"intent": "Post \"close because non reproducible\" for the merge request related to focus edge cases in a11yproject/a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1265",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "close because non reproducible"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 392,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "color ulitity",
"repo": "a11yproject.com",
"content": "Good idea"
},
"intent": "Post \"Good idea\" for the merge request related to color ulitity in a11yproject.com project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1071",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "Good idea"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 393,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
"instantiation_dict": {
"mr": "fixing the broken links",
"repo": "byteblaze/empathy-prompts",
"content": "lgtm"
},
"intent": "Post \"lgtm\" for the merge request related to fixing the broken links in byteblaze/empathy-prompts project",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/empathy-prompts/-/merge_requests/19",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"exact_match": "lgtm"
}
}
]
},
"intent_template_id": 348
},
{
"sites": [
"gitlab"
],
"task_id": 394,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "2019-nCov"
},
"intent": "Fork 2019-nCov.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/2019-nCov",
"locator": "",
"required_contents": {
"must_include": [
"2019-nCov"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 395,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "the Pytorch GAN repo with most stars"
},
"intent": "Fork the Pytorch GAN repo with most stars.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/PyTorch-GAN",
"locator": "",
"required_contents": {
"must_include": [
"Pytorch-GAN"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 396,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "ChatGPT"
},
"intent": "Fork ChatGPT.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/ChatGPT",
"locator": "",
"required_contents": {
"must_include": [
"ChatGPT"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 397,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "MetaSeq"
},
"intent": "Fork MetaSeq.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/metaseq",
"locator": "",
"required_contents": {
"must_include": [
"metaseq"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"gitlab"
],
"task_id": 398,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Fork {{repo}}.",
"instantiation_dict": {
"repo": "all source repos from Akilesh Kannan"
},
"intent": "Fork all source repos from Akilesh Kannan",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/SimCache",
"locator": "",
"required_contents": {
"must_include": [
"SimCache"
]
}
},
{
"url": "__GITLAB__/byteblaze/dots",
"locator": "",
"required_contents": {
"must_include": [
"dots"
]
}
},
{
"url": "__GITLAB__/byteblaze/CacheEval",
"locator": "",
"required_contents": {
"must_include": [
"CacheEval"
]
}
},
{
"url": "__GITLAB__/byteblaze/nvidia-patch",
"locator": "",
"required_contents": {
"must_include": [
"404"
]
}
},
{
"url": "__GITLAB__/byteblaze/viewgrades-scraper",
"locator": "",
"required_contents": {
"must_include": [
"404"
]
}
}
]
},
"intent_template_id": 352
},
{
"sites": [
"reddit"
],
"task_id": 399,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "I am a robot"
},
"intent": "Change my reddit bio to \"I am a robot\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "I am a robot"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 400,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Pro Python Developer with 20 years of Experience"
},
"intent": "Change my reddit bio to \"Pro Python Developer with 20 years of Experience\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Pro Python Developer with 20 years of Experience"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 401,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Seeking SDE positions"
},
"intent": "Change my reddit bio to \"Seeking SDE positions\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Seeking SDE positions"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 402,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Freelance Web Developer"
},
"intent": "Change my reddit bio to \"Freelance Web Developer\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Freelance Web Developer"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 403,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Change my reddit bio to \"{{content}}\"",
"instantiation_dict": {
"content": "Awesome Prompt Artist"
},
"intent": "Change my reddit bio to \"Awesome Prompt Artist\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/user/MarvelsGrantMan136",
"locator": "document.querySelector(\".user-bio__biography\").outerText",
"required_contents": {
"exact_match": "Awesome Prompt Artist"
}
}
]
},
"intent_template_id": 6
},
{
"sites": [
"reddit"
],
"task_id": 404,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "books"
},
"intent": "Upvote the newest post in books subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/books/124260/adults-reading-to-each-other-out-loud",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 405,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "DIY"
},
"intent": "Upvote the newest post in DIY subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/DIY/119019/how-can-i-bring-an-hdmi-cable-from-my-pc-downstairs-to-my-tv",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 406,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "future technology"
},
"intent": "Upvote the newest post in future technology subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/Futurology/119517/openai-ceo-it-s-not-funny-that-i-m-afraid-of-the-ai-we-re",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 407,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "deeplearning"
},
"intent": "Upvote the newest post in deeplearning subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/deeplearning/124993/meta-s-llama-weights-leaked-on-torrent-and-the-best-thing",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 408,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__",
"geolocation": null,
"intent_template": "Upvote the newest post in {{subreddit}} subreddit",
"instantiation_dict": {
"subreddit": "explain like im 5"
},
"intent": "Upvote the newest post in explain like im 5 subreddit",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/explainlikeimfive/39244/eli5-how-does-pinching-a-ribbon-and-sliding-your-finger",
"locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
"required_contents": {
"must_include": [
"vote vote--user-upvoted"
]
}
}
]
},
"intent_template_id": 22
},
{
"sites": [
"reddit"
],
"task_id": 409,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
"geolocation": null,
"intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the manager of the website",
"content_description": "thanks! I am a big fan of your website."
},
"intent": "Reply to the manager of the website in this post with \"thanks! I am a big fan of your website.\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/books/59421/-/comment/1235250",
"locator": "",
"required_contents": {
"must_include": [
"thanks! I am a big fan of your website."
]
}
}
]
},
"intent_template_id": 23
},
{
"sites": [
"reddit"
],
"task_id": 410,
"require_login": true,
"storage_state": "./.auth/reddit_state.json",
"start_url": "__REDDIT__/f/singularity/69404/this-is-how-chatgpt-sees-itself",
"geolocation": null,
"intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
"instantiation_dict": {
"position_description": "the first reply",
"content_description": "don't panic"
},
"intent": "Reply to the first reply in this post with \"don't panic\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__REDDIT__/f/singularity/69404/-/comment/1042264",
"locator": "",
"required_contents": {
"must_include": [
"don't panic"
]
}
}
]
},
"intent_template_id": 23
},
{
"sites": [
"gitlab"
],
"task_id": 411,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to MIT license.",
"instantiation_dict": {
"repo": "byteblaze/cloud-to-butt"
},
"intent": "Make the LICENSE of byteblaze/cloud-to-butt to MIT license.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/cloud-to-butt/-/blob/master/LICENSE.txt",
"locator": "",
"required_contents": {
"must_include": [
"MIT license"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 412,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to Apache License",
"instantiation_dict": {
"repo": "byteblaze/accessible-html-content-patterns"
},
"intent": "Make the LICENSE of byteblaze/accessible-html-content-patterns to Apache License",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/accessible-html-content-patterns/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"Apache License",
"http://www.apache.org/licenses/LICENSE-2.0"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 413,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to one that mandates all copies and derivative works to be under the same license",
"instantiation_dict": {
"repo": "byteblaze/a11y-syntax-highlighting"
},
"intent": "Make the LICENSE of byteblaze/a11y-syntax-highlighting to one that mandates all copies and derivative works to be under the same license",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"GENERAL PUBLIC LICENSE"
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 414,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Make the LICENSE of {{repo}} to MIT license.",
"instantiation_dict": {
"repo": "byteblaze/dotfiles"
},
"intent": "Make the LICENSE of byteblaze/dotfiles to MIT license.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/blob/main/LICENSE",
"locator": "",
"required_contents": {
"must_include": [
"MIT license",
"The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
]
}
}
]
},
"intent_template_id": 355
},
{
"sites": [
"gitlab"
],
"task_id": 415,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "verification functions"
},
"intent": "Go to the merge request on verification functions I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/a11y-webring.club/-/merge_requests/40",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["@davepgreene"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 416,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "wcag"
},
"intent": "Go to the merge request on wcag I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1270",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["Thank you"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 417,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"instantiation_dict": {
"topic": "404 link"
},
"intent": "Go to the merge request on 404 link I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1485",
"locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
"required_contents": {
"must_include": ["@Roshanjossey"]
}
}
]
},
"intent_template_id": 360
},
{
"sites": [
"gitlab"
],
"task_id": 418,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Busy"
},
"intent": "Set my gitlab status as Busy.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-title').textContent",
"required_contents": {
"must_include": ["Busy"]
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 419,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Enjoying life"
},
"intent": "Set my gitlab status as Enjoying life.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Enjoying life"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 420,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Playing Badminton"
},
"intent": "Set my gitlab status as Playing Badminton.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Playing Badminton"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 421,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Resting due to leg injury"
},
"intent": "Set my gitlab status as Resting due to leg injury.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Resting due to leg injury"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"gitlab"
],
"task_id": 422,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set my gitlab status as {{status}}.",
"instantiation_dict": {
"status": "Out of Office"
},
"intent": "Set my gitlab status as Out of Office.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.cover-status').lastChild.textContent",
"required_contents": {
"exact_match": "Out of Office"
}
}
]
},
"intent_template_id": 361
},
{
"sites": [
"shopping_admin"
],
"task_id": 423,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Mark all {{brand}} shirts on sale",
"instantiation_dict": {
"brand": "Hollister"
},
"intent": "Mark all Hollister shirts on sale",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/126/",
"locator": "document.querySelector('input[name=\"product[sale]\"]').value",
"required_contents": {
"exact_match": "1"
}
}
]
},
"intent_template_id": 237
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 424,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the place where Mr. Rogers was filmed"
},
"intent": "Find the page of the place where Mr. Rogers was filmed on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Pittsburgh"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 425,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the longest bridge in the Western hemisphere"
},
"intent": "Find the page of the longest bridge in the Western hemisphere on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Mackinac Bridge"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 426,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the place in Pennsylvania where a plane crashed during the September 11th attacks"
},
"intent": "Find the page of the place in Pennsylvania where a plane crashed during the September 11th attacks on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Somerset County"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 427,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the university that has most Turning Award winners"
},
"intent": "Find the page of the university that has most Turning Award winners on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Massachusetts Institute of Technology"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 428,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the undergrad college of the person who developed the Nash equilibrium"
},
"intent": "Find the page of the undergrad college of the person who developed the Nash equilibrium on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Carnegie Mellon University"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 429,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the colleges where The Chair was filmed in Pittsburgh"
},
"intent": "Find the page of the colleges where The Chair was filmed in Pittsburgh on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Chatham University"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"wikipedia",
"map"
],
"task_id": 430,
"require_login": true,
"storage_state": null,
"start_url": "__MAP__",
"geolocation": null,
"intent_template": "Find the page of {{description}} on the map.",
"instantiation_dict": {
"description": "the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh"
},
"intent": "Find the page of the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh on the map.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "last",
"locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
"required_contents": {
"must_include": [
"Washington & Jefferson College"
]
}
}
]
},
"intent_template_id": 371
},
{
"sites": [
"shopping"
],
"task_id": 431,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/tall-pink-taper-candles-4-piece-orange-colored-tapered-candles-gradient-candles-10-6-inches-tall-tie-dye-candle-set-large-dripless-long-burning-candlesticks-two-color-taper-candles-candlesticks.html |AND| __SHOPPING__/spaas-white-taper-candles-4-pack-10-inch-tall-candles-scent-free-premium-wax-candle-sticks-8-hour-long-burning-white-candlesticks-for-home-decoration-wedding-holiday-and-parties.html |AND| __SHOPPING__/white-starfish-wall-candle-sconces-set-of-2-beach-decor-ocean-themed-wall-mount-candleholders-nautical-style-beach-bathroom-decor-coastal-farmhouse-seashell-candle-holders.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"SPAAS White Taper Candles - 4 Pack |OR| 10 Inch Tall Candles, Scent-Free Premium Wax Candle Sticks |OR| 8 Hour Long Burning White Candlesticks for Home Decoration, Wedding, Holiday and Parties"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 432,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/ciclon-energy-drink-regular-24-cans-8-3oz.html |AND| __SHOPPING__/v8-energy-healthy-energy-drink-steady-energy-from-black-and-green-tea-pomegranate-blueberry-8-ounce-can-pack-of-24.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"V8 +Energy, Healthy Energy Drink, Steady Energy from Black and Green Tea, Pomegranate Blueberry, 8 Ounce Can ,Pack of 24"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 433,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/tazrigo-5pcs-white-dental-resin-brush-pens-dental-shaping-silicone-tooth-tool.html |AND| __SHOPPING__/stylus-pens-for-touch-screens-2-pcs-universal-stylus-2-in-1-2022-updated-touch-screen-pens-for-all-touch-screens-cell-phones-tablets-laptops-with-6-replacement-tips-4-discstips-2-fiber-tips.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"Tazrigo 5pcs White Dental Resin Brush Pens Dental Shaping Silicone Tooth Tool"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 434,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/3-pairs-ruffle-socks-lace-ankle-socks-for-girls-frilly-socks-women-decorative.html |AND| __SHOPPING__/viviki-women-glitter-socks-ultrathin-transparent-tulle-lace-socks-no-show-ankle-crew-socks-3-pack.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"VIVIKI Women Glitter Socks Ultrathin Transparent Tulle Lace Socks - No Show Ankle Crew Socks 3 Pack"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 435,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__/35-ft-hdmi-cable-gearit-pro-series-hdmi-cable-35-feet-high-speed-ethernet-4k-resolution-3d-video-and-arc-audio-return-channel-hdmi-cable-white.html |AND| __SHOPPING__/dp-to-hdmi-cable-6ft-2-pack-fosmon-gold-plated-displayport-to-hdmi-cable-1080p-full-hd-for-pcs-to-hdtv-monitor-projector-with-hdmi-port.html",
"geolocation": null,
"intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"instantiation_dict": {},
"intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/checkout/cart",
"locator": "",
"required_contents": {
"must_include": [
"DP to HDMI Cable 6FT (2 Pack), Fosmon Gold Plated Displayport to HDMI Cable 1080p Full HD for PCs to HDTV, Monitor, Projector with HDMI Port"
]
}
}
]
},
"intent_template_id": 145
},
{
"sites": [
"shopping"
],
"task_id": 436,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a mattress foundation",
"time": "around Feb or March 2023"
},
"intent": "I previously ordered some a mattress foundation around Feb or March 2023 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B07DFJ5XKH"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 437,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a table lamp",
"time": "in May 2023"
},
"intent": "I previously ordered some a table lamp in May 2023 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B072XS3F6W"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 438,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a TV stand",
"time": "sometime around sep 2022"
},
"intent": "I previously ordered some a TV stand sometime around sep 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B08PVHRRB7"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 439,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a cat t-shirt",
"time": "during 2022"
},
"intent": "I previously ordered some a cat t-shirt during 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B0844BWS76"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"shopping"
],
"task_id": 440,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
"instantiation_dict": {
"product": "a make up removal kit",
"time": "during summer 2022"
},
"intent": "I previously ordered some a make up removal kit during summer 2022 and later cancelled. Can you reorder it for me?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "func:shopping_get_latest_order_url()",
"locator": "document.querySelector(\".order-details-items.ordered\").outerText",
"required_contents": {
"must_include": [
"B0738JQG6Q"
]
}
}
]
},
"intent_template_id": 156
},
{
"sites": [
"gitlab"
],
"task_id": 441,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "GIVE ME SPACE"
},
"intent": "Update the project site's title to \"GIVE ME SPACE\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>GIVE ME SPACE</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 442,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Welcome to my site"
},
"intent": "Update the project site's title to \"Welcome to my site\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Welcome to my site</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 443,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Not an interesting site"
},
"intent": "Update the project site's title to \"Not an interesting site\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Not an interesting site</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 444,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Title Wanted"
},
"intent": "Update the project site's title to \"Title Wanted\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Title Wanted</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 445,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__/byteblaze/gimmiethat.space",
"geolocation": null,
"intent_template": "Update the project site's title to \"{{title}}\"",
"instantiation_dict": {
"title": "Hello"
},
"intent": "Update the project site's title to \"Hello\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
"locator": "",
"required_contents": {
"must_include": [
"<title>Hello</title>"
]
}
}
]
},
"intent_template_id": 308
},
{
"sites": [
"gitlab"
],
"task_id": 446,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
"instantiation_dict": {
"repo": "a11yproject",
"issue": 404,
"account": "Roshanjossey"
},
"intent": "Assign the issue regarding 404 in a11yproject to Roshanjossey.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Roshanjossey",
"locator": "",
"required_contents": {
"must_include": [
"404s, bad host, timeouts, bad urls for URLs linked from website"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 447,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
"instantiation_dict": {
"repo": "a11y-webring.club",
"issue": "linking to an accessibility statement",
"account": "Rohan"
},
"intent": "Assign the issue regarding linking to an accessibility statement in a11y-webring.club to Rohan.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Seirdy",
"locator": "",
"required_contents": {
"must_include": [
"linking to an accessibility statement"
]
}
}
]
},
"intent_template_id": 999
},
{
"sites": [
"gitlab"
],
"task_id": 448,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "https://egg.tart.com"
},
"intent": "set the homepage URL on my GitLab profile to https://egg.tart.com",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "egg.tart.com"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 449,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "https://helloworld.xyz"
},
"intent": "set the homepage URL on my GitLab profile to https://helloworld.xyz",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "helloworld.xyz"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 450,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "a11yproject.contributor.me"
},
"intent": "set the homepage URL on my GitLab profile to a11yproject.contributor.me",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "a11yproject.contributor.me"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 451,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "www.byteblaze.com"
},
"intent": "set the homepage URL on my GitLab profile to www.byteblaze.com",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "www.byteblaze.com"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"gitlab"
],
"task_id": 452,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "set the homepage URL on my GitLab profile to {{url}}",
"instantiation_dict": {
"url": "byteblaze.github.io"
},
"intent": "set the homepage URL on my GitLab profile to byteblaze.github.io",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze",
"locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
"required_contents": {
"exact_match": "byteblaze.github.io"
}
}
]
},
"intent_template_id": 331
},
{
"sites": [
"shopping_admin"
],
"task_id": 453,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Teton pullover hoodie"
},
"intent": "Disable Teton pullover hoodie from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/78/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 454,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Ryker Tee Crew Neck"
},
"intent": "Disable Ryker Tee Crew Neck from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/478/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 455,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "lHelios Endurance Tank"
},
"intent": "Disable lHelios Endurance Tank from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/676/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 456,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Cora Pant"
},
"intent": "Disable Cora Pant from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1840/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 457,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
"instantiation_dict": {
"product": "Karmen yoga pants"
},
"intent": "Disable Karmen yoga pants from the site, they are facing some quality issues.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1819/",
"locator": "document.querySelector('[name=\"product[status]\"').value",
"required_contents": {
"exact_match": "2"
}
}
]
},
"intent_template_id": 242
},
{
"sites": [
"shopping_admin"
],
"task_id": 458,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "$5",
"action": "Reduce"
},
"intent": "Reduce the price of this product by $5",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "27.00"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 459,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "10%",
"action": "Reduce"
},
"intent": "Reduce the price of this product by 10%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "62.10"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 460,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "15%",
"action": "Reduce"
},
"intent": "Reduce the price of this product by 15%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "38.25"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 461,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "$11.5",
"action": "Increase"
},
"intent": "Increase the price of this product by $11.5",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "29.50"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 462,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "10%",
"action": "Increase"
},
"intent": "Increase the price of this product by 10%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "19.80"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 463,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
"geolocation": null,
"intent_template": "{{action}} the price of this product by {{amount}}",
"instantiation_dict": {
"amount": "15%",
"action": "Increase"
},
"intent": "Increase the price of this product by 15%",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
"locator": "document.querySelector('[name=\"product[price]\"').value",
"required_contents": {
"exact_match": "36.80"
}
}
]
},
"intent_template_id": 247
},
{
"sites": [
"shopping_admin"
],
"task_id": 464,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Update the product description of {{product}} to highlight the real user positive reviews by quoting the comments",
"instantiation_dict": {
"product": "Antonia Racer Tank"
},
"intent": "Update the product description of Antonia Racer Tank to highlight the real user positive reviews by quoting the comments",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/../antonia-racer-tank.html",
"locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
"required_contents": {
"must_include": [
"This is in regular rotation at the gym",
"Its colorful and looks kinda cute under my exercise tanks",
"it's very stylish for yoga or something else low impact"
]
}
}
]
},
"intent_template_id": 251
},
{
"sites": [
"shopping"
],
"task_id": 465,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
},
"intent": "Add Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 466,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
},
"intent": "Add 2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 467,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
},
"intent": "Add HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 468,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
},
"intent": "Add DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping"
],
"task_id": 469,
"require_login": true,
"storage_state": "./.auth/shopping_state.json",
"start_url": "__SHOPPING__",
"geolocation": null,
"intent_template": "Add {{product}} to my wish list",
"instantiation_dict": {
"product": "Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
},
"intent": "Add Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes to my wish list",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING__/wishlist/",
"locator": "document.querySelector('.products-grid.wishlist').outerText",
"required_contents": {
"must_include": [
"Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
]
}
}
]
},
"intent_template_id": 186
},
{
"sites": [
"shopping_admin"
],
"task_id": 470,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "302"
},
"intent": "Cancel order 302",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 471,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "307"
},
"intent": "Cancel order 307",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/307/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 472,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "299"
},
"intent": "Cancel order 299",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/299/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 473,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "301"
},
"intent": "Cancel order 301",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/301/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"shopping_admin"
],
"task_id": 474,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Cancel order {{id}}",
"instantiation_dict": {
"id": "305"
},
"intent": "Cancel order 305",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/305/",
"locator": "document.querySelector(\"#order_status\").outerText",
"required_contents": {
"exact_match": "Canceled"
}
}
]
},
"intent_template_id": 257
},
{
"sites": [
"gitlab"
],
"task_id": 475,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "chatgpt_plugin"
},
"intent": "Set up a new, empty repository with the name chatgpt_plugin?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/chatgpt_plugin",
"locator": "",
"required_contents": {
"must_include": [
"chatgpt_plugin"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 476,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_llm_reading"
},
"intent": "Set up a new, empty repository with the name awesome_llm_reading?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_llm_reading",
"locator": "",
"required_contents": {
"must_include": [
"awesome_llm_reading"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 477,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_program_aided_reasoning"
},
"intent": "Set up a new, empty repository with the name awesome_program_aided_reasoning?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_program_aided_reasoning",
"locator": "",
"required_contents": {
"must_include": [
"awesome_program_aided_reasoning"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 478,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "webagent"
},
"intent": "Set up a new, empty repository with the name webagent?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/webagent",
"locator": "",
"required_contents": {
"must_include": [
"webagent"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 479,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Set up a new, empty repository with the name {{project_name}}?",
"instantiation_dict": {
"project_name": "awesome_webagent"
},
"intent": "Set up a new, empty repository with the name awesome_webagent?",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/awesome_webagent",
"locator": "",
"required_contents": {
"must_include": [
"awesome_webagent"
]
}
}
]
},
"intent_template_id": 292
},
{
"sites": [
"gitlab"
],
"task_id": 480,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}}",
"instantiation_dict": {
"collaborator_account_list": "yjlou",
"repo": "solarized-prism-theme"
},
"intent": "Invite yjlou as collaborator to solarized-prism-theme",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/solarized-prism-theme/-/project_members",
"locator": "",
"required_contents": {
"must_include": [
"yjlou"
]
}
}
]
},
"intent_template_id": 293
},
{
"sites": [
"gitlab"
],
"task_id": 481,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Abishek"
},
"intent": "Abishek wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 482,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "yjlou"
},
"intent": "yjlou wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 483,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Koushik"
},
"intent": "Koushik wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'koush')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 484,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Jakub Klinkovsk\u00fd"
},
"intent": "Jakub Klinkovsk\u00fd wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"gitlab"
],
"task_id": 485,
"require_login": true,
"storage_state": "./.auth/gitlab_state.json",
"start_url": "__GITLAB__",
"geolocation": null,
"intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"instantiation_dict": {
"name": "Vinta"
},
"intent": "Vinta wants to check my dotfile configurations. Please invite him to the repo as a guest.",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
"locator": "func:gitlab_get_project_memeber_role(__page__, 'vinta')",
"required_contents": {
"must_include": [
"Guest"
]
}
}
]
},
"intent_template_id": 294
},
{
"sites": [
"shopping_admin"
],
"task_id": 486,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "404 Not Found",
"heading": "Bruh bro you clicked the wrong page"
},
"intent": "Change the page title of \"404 Not Found\" page on my site to \"Bruh bro you clicked the wrong page\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/1/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Bruh bro you clicked the wrong page"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 487,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Enable Cookies",
"heading": "Cookie monster coming to your place"
},
"intent": "Change the page title of \"Enable Cookies\" page on my site to \"Cookie monster coming to your place\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/3/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Cookie monster coming to your place"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 488,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Home Page",
"heading": "This is the home page!! Leave here!!"
},
"intent": "Change the page title of \"Home Page\" page on my site to \"This is the home page!! Leave here!!\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/2/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "This is the home page!! Leave here!!"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 489,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "Privacy Policy",
"heading": "No privacy policy is needed in this dystopian world"
},
"intent": "Change the page title of \"Privacy Policy\" page on my site to \"No privacy policy is needed in this dystopian world\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/4/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "No privacy policy is needed in this dystopian world"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 490,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
"instantiation_dict": {
"old-heading": "About us",
"heading": "Secret"
},
"intent": "Change the page title of \"About us\" page on my site to \"Secret\".",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/5/",
"locator": "document.querySelector('input[name=\"title\"').value",
"required_contents": {
"exact_match": "Secret"
}
}
]
},
"intent_template_id": 275
},
{
"sites": [
"shopping_admin"
],
"task_id": 491,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Sarah Miller",
"message": "the order is ready to be shipped soon!"
},
"intent": "Notify Sarah Miller in their most recent pending order with message \"the order is ready to be shipped soon!\"",
"require_reset": false,
"eval": {
"eval_types": [
"string_match"
],
"reference_answers": {
"fuzzy_match": "N/A"
},
"reference_url": "",
"program_html": [],
"string_note": "System message: We cannot add order history."
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 492,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Jane Doe",
"message": "sorry we are out of stock, please reorder"
},
"intent": "Notify Jane Doe in their most recent pending order with message \"sorry we are out of stock, please reorder\"",
"require_reset": false,
"eval": {
"eval_types": [
"program_html"
],
"reference_answers": null,
"reference_url": "",
"program_html": [
{
"url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
"locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
"required_contents": {
"exact_match": "sorry we are out of stock, please reorder"
}
}
]
},
"intent_template_id": 280
},
{
"sites": [
"shopping_admin"
],
"task_id": 493,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
"instantiation_dict": {
"name": "Grace Nguyen",
"message": "sorry we are bankrupt, please contact our customer service for refund"
},
"intent": "Notify Grace Nguyen in their most recent pending order with message \"sorry we are bankrupt, please contact our customer service for refund\"",
"require_reset": false,