{action['raw_prediction']}{repr(action)}{action2str(action, action_set_tag, node_content)}{action['raw_prediction']}{repr(action)}{action2str(action, action_set_tag, node_content)}{action['raw_prediction']}{repr(action)}{action2str(action, action_set_tag, node_content)}{_config_str}\n"
+ task_id = _config["task_id"]
+
+ self.action_set_tag = action_set_tag
+
+ self.render_file = open(
+ Path(result_dir) / f"render_{task_id}.html", "a+"
+ )
+ self.render_file.truncate(0)
+ # write init template
+ self.render_file.write(HTML_TEMPLATE.format(body=f"{_config_str}"))
+ self.render_file.read()
+ self.render_file.flush()
+
+ def render(
+ self,
+ action: Action,
+ state_info: StateInfo,
+ meta_data: dict[str, Any],
+ render_screenshot: bool = False,
+ ) -> None:
+ """Render the trajectory"""
+ # text observation
+ observation = state_info["observation"]
+ text_obs = observation["text"]
+ info = state_info["info"]
+ new_content = f"New Page
\n" + new_content += f"URL: {state_info['info']['page'].url}
\n" + new_content += f"{text_obs}\n"
+
+ if render_screenshot:
+ # image observation
+ img_obs = observation["image"]
+ image = Image.fromarray(img_obs)
+ byte_io = io.BytesIO()
+ image.save(byte_io, format="PNG")
+ byte_io.seek(0)
+ image_bytes = base64.b64encode(byte_io.read())
+ image_str = image_bytes.decode("utf-8")
+ new_content += f"
\n"
+
+ # meta data
+ new_content += f"[\W\w]*?', '', ctx)
+ ctx = re.sub('[\W\w]*?', '', ctx)
+ ctx = '' if ctx is None else re.sub(r'\s+', ' ', ctx).strip()
+ dom_tree = html.fromstring(ctx)
+ return dom_tree
+
+ @staticmethod
+ def get_root(tree: html.HtmlElement) -> html.HtmlElement:
+ node = tree.xpath('//*')[0]
+ while True:
+ parent = node.getparent()
+ if parent is None:
+ break
+ node = parent
+ return node
+
+ def get_node_by_bid(self, tree: html.HtmlElement, bid: str) -> html.HtmlElement:
+ nodes = tree.xpath(f'//*[@{self.id_attr}="{bid}"]')
+ if len(nodes) == 0:
+ return None
+ return nodes[0]
+
+ def id_label_converter(self, label: str) -> str:
+ return self.bids2label.get(label, '')
+
+ def id_xpath_converter(self, label: str) -> str:
+ return self.bids2xpath.get(label, '')
+
+ def mark_id(self) -> None:
+ root = self.get_root(self.dom_tree)
+ _, i2xpath, used_labels = get_xpath_top_down(root, self.id_attr, self.label_attr)
+ self.used_labels = used_labels
+ self.bids2xpath = i2xpath
+
+ def parse(self, root: html.HtmlElement, keep: list[str], obs: list[str], parent_chain: bool=False, get_new_label: bool=False) -> dict[str]:
+ def get_text(str: str) -> str:
+ return '' if str is None else str.strip()[:500]
+
+ def check_attr(attr: str, node: html.HtmlElement) -> bool:
+ tag = node.tag
+ if (
+ ( attr == 'role' and node.attrib.get(attr, '') in ['presentation', 'none', 'link'] )
+ or ( attr == 'type' and node.attrib.get(attr, '') == 'hidden' )
+ # or ( attr == 'value' and tag in ['option'] )
+ ):
+ return False
+ return True
+
+ def is_visible(node: html.HtmlElement, bid: str) -> bool:
+ if self.dataset == 'mind2web':
+ bound = node.attrib.get('bounding_box_rect', None)
+ self.rect[bid] = rect2tuple(bound)
+
+ if not self.use_position:
+ return True
+
+ rect = self.rect.get(bid, None)
+ if rect is None:
+ return False
+
+ if self.window_size is None:
+ return True
+
+ # get window size
+ wx, wy, ww, wh = self.window_size
+ x, y, w, h = rect
+ if x + w < wx or x > wx + ww or y + h < wy or y > wy + wh:
+ return False
+
+ return True
+
+ def _dfs(node: html.HtmlElement, keep: list[str]=[], obs: list[str]=[],
+ parent_chain: bool=False, get_new_label: bool=False, par_keep: bool=False) -> (str, dict[str]):
+ # basic information
+ bid = node.attrib.get(self.id_attr, '')
+ tag = node.tag
+ label = node.attrib.get(self.label_attr, '')
+
+ # element which is keeped equivalent to visible
+ visible = is_visible(node, bid)
+ in_keep_list = bid in keep
+ in_obs_list = (bid in obs or len(label) > 0) and visible
+ keep_element = in_keep_list or in_obs_list or visible or par_keep
+
+ # mark label
+ bids2label, labeled_elems = {}, []
+ have_label = False
+ if in_keep_list or in_obs_list:
+ if label is None or len(label) == 0 or get_new_label:
+ label = self.identifier.generate()
+ node.attrib[self.label_attr] = label
+ bids2label[bid] = label
+ bids2label[label] = bid
+ have_label = True
+
+ # get text or alt_text of current element
+ text = get_text(node.text)
+
+ classes = {}
+ # keep attributes if needed
+ keep_all_attrs = len(self.keep_attrs) == 0
+ keep_attrs = node.attrib.keys() if keep_all_attrs else self.keep_attrs
+
+ # traverse attributes
+ for attr in keep_attrs:
+ if attr not in node.attrib or not check_attr(attr, node):
+ continue
+ if attr in [self.id_attr, self.label_attr]:
+ continue
+ val = get_text(node.attrib[attr])
+ if len(val) > 0 or keep_all_attrs:
+ classes[attr] = val
+
+ have_text = len(text) > 0 or len(classes) > 0
+
+ parts = []
+ clickable_count = 0
+ children = node.getchildren()
+ for child in children:
+ cres, cmsg = _dfs(child, keep, obs, parent_chain, get_new_label)
+ clickable_count += 1 if cmsg.get('have_clickable', False) else 0
+ bids2label.update(cmsg.get('bids2label', {}))
+ labeled_elems.extend(cmsg.get('label_element', []))
+ if len(cres) != 0:
+ parts.append(cres)
+
+ dom = self.prompt.subtree_constructor(parts)
+
+ # remove if all children are text
+ keep_as_all_text = (dom.count('<') == dom.count(' 0
+ if keep_as_all_text:
+ matches = re.findall(r']+) >', dom)
+ dom = self.prompt.subtree_constructor(matches)
+
+ keep_element = keep_element and (clickable_count > 1 or have_text or have_label or keep_as_all_text)
+ keep_as_parent = len(dom) > 0 and parent_chain
+ if in_keep_list or keep_element or keep_as_parent:
+ dom = self.prompt.prompt_constructor(tag, label, text, dom, classes)
+
+ if have_label:
+ labeled_elems.append(bid)
+
+ control_msg = {
+ 'have_clickable': bool(clickable_count or have_text),
+ 'bids2label': bids2label,
+ 'label_element': labeled_elems,
+ }
+
+ return dom, control_msg
+
+ dom, cmsg = _dfs(root, keep, obs, parent_chain, get_new_label)
+ return dom, cmsg
+
+ def parse_tree(self) -> dict[str]:
+ # start from here
+ stt = time.time()
+ root = self.get_root(self.dom_tree)
+ dom, cmsg = self.parse(root, self.keep, self.obs, self.parent_chain, self.get_new_label)
+ self.bids2label = cmsg.get('bids2label', {})
+ self.keep = list(set(self.keep + cmsg.get('label_element', [])))
+
+ obj = {
+ 'html': dom,
+ 'parse_time': time.time() - stt
+ }
+
+ return obj
+
+ # From mind2web, https://github.com/OSU-NLP-Group/Mind2Web/blob/main/src/data_utils/dom_utils.py
+ def get_keep_elements(self, tree: html.HtmlElement, keep: list[str], max_depth: int, max_children: int,
+ max_sibling: int, dfs_count: int=1, keep_parent: bool=False) -> list[str]:
+ def get_anscendants(node: html.HtmlElement, max_depth: int, current_depth: int=0) -> list[str]:
+ if current_depth > max_depth:
+ return []
+
+ anscendants = []
+ parent = node.getparent()
+ if parent is not None:
+ anscendants.append(parent)
+ anscendants.extend(get_anscendants(parent, max_depth, current_depth + 1))
+
+ return anscendants
+
+ def get_descendants(node: html.HtmlElement, max_depth: int, current_depth: int=0) -> list[str]:
+ if current_depth > max_depth:
+ return []
+
+ descendants = []
+ for child in node:
+ descendants.append(child)
+ descendants.extend(get_descendants(child, max_depth, current_depth + 1))
+
+ return descendants
+
+ to_keep = set(copy.deepcopy(keep))
+ nodes_to_keep = set()
+
+ for _ in range(max(1, dfs_count)):
+ for bid in to_keep:
+ candidate_node = self.get_node_by_bid(tree, bid)
+ if candidate_node is None:
+ continue
+
+ nodes_to_keep.add(candidate_node.attrib[self.id_attr])
+ # get all ancestors or with max depth
+ nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in get_anscendants(candidate_node, max_depth)])
+
+ # get descendants with max depth
+ nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in get_descendants(candidate_node, max_depth)][:max_children])
+ # get siblings within range
+ parent = candidate_node.getparent()
+ if parent is None:
+ continue
+
+ siblings = [x for x in parent.getchildren() if x.tag != 'text']
+ if candidate_node not in siblings:
+ continue
+
+ idx_in_sibling = siblings.index(candidate_node)
+ nodes_to_keep.update([x.attrib.get(self.id_attr, '')
+ for x in siblings[max(0, idx_in_sibling - max_sibling) : idx_in_sibling + max_sibling + 1]])
+
+ max_children = int(max_children * 0.5)
+ max_depth = int(max_depth * 0.5)
+ max_sibling = int(max_sibling * 0.7)
+
+ to_keep = copy.deepcopy(nodes_to_keep)
+
+ if keep_parent:
+ for bid in keep:
+ candidate_node = self.get_node_by_bid(tree, bid)
+ if candidate_node is None:
+ continue
+ nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in candidate_node.xpath("ancestor::*")])
+
+ return list(nodes_to_keep)
+
+ def prune(self, tree: html.HtmlElement, nodes_to_keep: list[str]) -> html.HtmlElement:
+ # remove nodes not in nodes_to_keep
+ for node in tree.xpath('//*')[::-1]:
+ if node.tag != 'text':
+ is_keep = node.attrib.get(self.id_attr, '') in nodes_to_keep
+ is_candidate = node.attrib.get(self.id_attr, '') in self.keep
+ else:
+ is_keep = (node.getparent().attrib.get(self.id_attr, '') in nodes_to_keep)
+ is_candidate = (node.getparent().attrib.get(self.id_attr, '') in self.keep)
+
+ if not is_keep and node.getparent() is not None:
+ # insert all children into parent
+ for child in node.getchildren():
+ node.addprevious(child)
+ node.getparent().remove(node)
+ else:
+ # if not is_candidate or node.tag == 'text':
+ # node.attrib.pop(self.id_attr, None)
+ if (
+ len(node.attrib) == 0
+ and not any([x.tag == 'text' for x in node.getchildren()])
+ and node.getparent() is not None
+ and node.tag != "text"
+ and len(node.getchildren()) <= 1
+ ):
+ # insert all children into parent
+ for child in node.getchildren():
+ node.addprevious(child)
+ node.getparent().remove(node)
+
+ return tree
+
+ def prune_tree(self, dfs_count: int=1, max_depth: int=3, max_children: int=30,
+ max_sibling: int=3, keep_parent: bool=False) -> None:
+ # clone the tree
+ new_tree = copy.deepcopy(self.dom_tree)
+ nodes_to_keep = self.get_keep_elements(new_tree, self.keep, max_depth, max_children, max_sibling, dfs_count, keep_parent)
+ new_tree = self.prune(new_tree, nodes_to_keep)
+
+ self.dom_tree = new_tree
+
+ def get_segment(self, bid: str) -> str:
+ # clone the tree
+ new_tree = copy.deepcopy(self.dom_tree)
+ nodes_to_keep = self.get_keep_elements(new_tree, [bid], 0, 2, 1)
+ new_tree = self.prune(new_tree, nodes_to_keep)
+ dom, _ = self.parse(new_tree, self.keep, [], False)
+ return dom
+
+ def get_rect_data(self, bids: list[str]) -> list[dict[str]]:
+ res = []
+ for bid in bids:
+ label = self.bids2label.get(bid, '')
+ rect = self.rect.get(bid, None)
+ res.append({
+ 'bid': bid,
+ 'label': label,
+ 'rect': rect
+ })
+ return res
+
\ No newline at end of file
diff --git a/browser_env/html_tools/identifier.py b/browser_env/html_tools/identifier.py
new file mode 100644
index 0000000..793ebed
--- /dev/null
+++ b/browser_env/html_tools/identifier.py
@@ -0,0 +1,64 @@
+import secrets
+
+class IdentifierTool:
+ def __init__(self, method: str='order', existing_labels: dict[str]={}) -> None:
+ self.methods = {
+ 'order': self.get_identifier_in_order,
+ 'random': self.get_random_identifier,
+ }
+
+ if method is None:
+ method = 'order'
+
+ self.func = self.methods.get(method, None)
+ self.name = method
+ if self.func is None:
+ raise ValueError(f'Invalid method for identifier: {method}')
+
+ self.reset(existing_labels)
+
+ def reset(self, exists: dict[str]={}) -> None:
+ self.identifier = -1
+ self.exists = {} if exists is None else exists
+
+ def get_identifier_in_order(self) -> str:
+ def id2str(id: int) -> str:
+ if id < 26:
+ return chr(id + 65)
+ id -= 26
+ c0 = id // 676
+ c1 = (id // 26) % 26
+ c2 = id % 26
+ label = f'{chr(c1 + 65)}{chr(c2 + 65)}'
+ return label if c0 == 0 else f'{chr(c0 + 64)}{label}'
+
+ self.identifier += 1
+ label = id2str(self.identifier)
+
+ while label in self.exists:
+ self.identifier += 1
+ label = id2str(self.identifier)
+
+ self.exists[label] = True
+ return label
+
+ def get_random_identifier(self) -> str:
+ secret_generator = secrets.SystemRandom()
+
+ def get_random_label(n: int=2) -> str:
+ tmp = ''
+ for _ in range(n):
+ tmp += chr(secret_generator.randint(65, 90))
+ return tmp
+
+ wc = 3 if len(self.exists) > 280 else 2
+
+ label = get_random_label(wc)
+ while label in self.exists:
+ label = get_random_label(wc)
+
+ self.exists[label] = True
+ return label
+
+ def generate(self):
+ return self.func()
\ No newline at end of file
diff --git a/browser_env/html_tools/prompt.py b/browser_env/html_tools/prompt.py
new file mode 100644
index 0000000..38d6b94
--- /dev/null
+++ b/browser_env/html_tools/prompt.py
@@ -0,0 +1,97 @@
+from .configs import prompts
+
+class HtmlPrompt:
+ def __init__(self, prompt: str='') -> None:
+ prompt = self.extract(prompt, 'xml')
+ if prompt not in prompts:
+ raise Exception('Unknown prompt: ' + prompt)
+
+ constructors = {
+ 'refine': self.normal_prompt_constructor,
+ 'xml': self.normal_prompt_constructor,
+ 'new_data': self.new_data_prompt_constructor,
+ }
+
+ self.name = prompt
+ self.prompt = prompts[prompt]
+ self.constructor = constructors[prompt]
+
+ @staticmethod
+ def extract(data, default=''):
+ return data if data is not None else default
+
+ def subtree_constructor(self, subtree: list[str]=[]) -> str:
+ return self.prompt['subtree_splitter'].join(subtree)
+
+ def normal_prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+ def add_prefix(data, prefix):
+ return prefix + data if len(data) > 0 else ''
+
+ tag = self.extract(tag)
+ label = self.extract(label)
+ content = self.extract(content)
+ subtree_str = self.extract(subtree_str, '')
+ class_dict = self.extract(class_dict, {})
+
+ label_str = ''
+ if len(label) > 0:
+ label_str = self.prompt['label'].format(label=label)
+
+ classes = []
+ values = set()
+ for key, val in class_dict.items():
+ if val in values:
+ continue
+ values.add(val)
+ classes.append(self.prompt['attr'].format(key=key, attr=val))
+ classes_str = self.prompt['attr_splitter'].join(classes)
+
+ content_splitter = ' ' if len(classes_str) == 0 else self.prompt['attr_splitter']
+ classes_str = add_prefix(classes_str, ' ')
+ content_str = add_prefix(content, content_splitter)
+ subtree_str = add_prefix(subtree_str, ' ')
+
+ return self.prompt['dom'].format(tag=tag, label=label_str, attr=classes_str, content=content_str, subtree=subtree_str)
+
+ def new_data_prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+ def add_prefix(data, prefix):
+ return prefix + data if len(data) > 0 else ''
+
+ tag = self.extract(tag)
+ label = self.extract(label)
+ content = self.extract(content)
+ subtree_str = self.extract(subtree_str, '')
+ class_dict = self.extract(class_dict, {})
+
+ label_str = ''
+ if len(label) > 0:
+ label_str = self.prompt['label'].format(label=label)
+
+ classes = []
+ values = set()
+
+ message = []
+ for key, val in class_dict.items():
+ if val == '':
+ message.append(key)
+ continue
+ if val in values:
+ continue
+ values.add(val)
+ classes.append(self.prompt['attr'].format(key=key, attr=val))
+
+ if len(message) > 0:
+ message_str = ' '.join(message)
+ classes.append(self.prompt['attr'].format(key='message', attr=message_str))
+
+ classes_str = self.prompt['attr_splitter'].join(classes)
+
+ content_splitter = ' ' if len(classes_str) == 0 else self.prompt['attr_splitter']
+ classes_str = add_prefix(classes_str, ' ')
+ content_str = add_prefix(content, content_splitter)
+ subtree_str = add_prefix(subtree_str, ' ')
+
+ return self.prompt['dom'].format(tag=tag, label=label_str, attr=classes_str, content=content_str, subtree=subtree_str)
+
+ def prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+ return self.constructor(tag, label, content, subtree_str, class_dict)
\ No newline at end of file
diff --git a/browser_env/html_tools/utils.py b/browser_env/html_tools/utils.py
new file mode 100644
index 0000000..c3c71fb
--- /dev/null
+++ b/browser_env/html_tools/utils.py
@@ -0,0 +1,101 @@
+from lxml import html
+def get_xpath_top_down(element: html.HtmlElement, id_column: str='temp_id', label_column: str='temp_clickable_label', path: str='', order: int=0,
+ in_svg: bool=False, temp_id: int=0) -> tuple[int, dict[str, str], dict[str]]:
+ used_labels, i2xpath = {}, {}
+ # path
+ tag = element.tag.lower()
+ in_svg = in_svg or (tag == 'svg')
+
+ if not in_svg and 'id' in element.attrib:
+ node_id = element.attrib['id']
+ path = f'//*[@id="{node_id}"]'
+ else:
+ suffix = f'[{order}]' if order > 0 else ''
+ prefix = f'*[name()="{tag}"]' if in_svg else tag
+ path = path + '/' + prefix + suffix
+
+ # add temp id
+ element.attrib[id_column] = str(temp_id)
+ ori_label = element.attrib.get(label_column, '')
+ if ori_label != '':
+ used_labels[ori_label] = True
+
+ bid = str(temp_id)
+ i2xpath[bid] = path
+ i2xpath[path] = bid
+ i2xpath[f'xpath/{path}'] = bid
+ i2xpath[f'xpath=/{path}'] = bid
+
+ temp_id += 1
+
+ # traverse node
+ children = element.getchildren()
+ tag_dict = {}
+ id_list = []
+ for child in children:
+ ctag = child.tag.lower()
+ if ctag not in tag_dict:
+ tag_dict[ctag] = 0
+ tag_dict[ctag] += 1
+ id_list.append(tag_dict[ctag])
+
+ for cid, child in zip(id_list, children):
+ ctag = child.tag.lower()
+ cod = cid if tag_dict[ctag] > 1 else 0
+ temp_id, i2x, ulabels = get_xpath_top_down(child, id_column, label_column, path, cod, in_svg, temp_id)
+ i2xpath.update(i2x)
+ used_labels.update(ulabels)
+
+ return temp_id, i2xpath, used_labels
+
+def print_html_object(obj: str='') -> str:
+ tab_cnt = 0
+ result, content, sep = '', '', ''
+ last_is_left, last_is_right = False, False
+ for ch in obj:
+ if ch == '<':
+ result += '\n'
+ if len(content.strip()) > 0:
+ result += sep + content.strip() + '\n'
+ result += sep + '<'
+
+ tab_cnt += 1
+ sep = ' ' * tab_cnt
+
+ content = ''
+ last_is_right = False
+ last_is_left = True
+ elif ch == '>':
+ if last_is_left:
+ result += content
+ else:
+ if last_is_right:
+ result += '\n'
+ if len(content.strip()) > 0:
+ result += sep + content.strip() + '\n'
+
+ tab_cnt -= 1
+ sep = ' ' * tab_cnt
+
+ if not last_is_left:
+ result += sep
+
+ result += '>'
+ content = ''
+
+ last_is_right = True
+ last_is_left = False
+ else:
+ content += ch
+
+ return result
+
+def rect2tuple(rect: str) -> tuple[int, int, int, int]:
+ if rect is None or type(rect) != type('str'):
+ return None
+ rect = rect.strip()
+ if rect.count(',') != 3:
+ return None
+ rect = rect.split(',')
+ rect = [float(r) for r in rect]
+ return tuple(rect)
\ No newline at end of file
diff --git a/browser_env/processors.py b/browser_env/processors.py
new file mode 100644
index 0000000..e7daadd
--- /dev/null
+++ b/browser_env/processors.py
@@ -0,0 +1,1125 @@
+import json
+import lxml
+import re
+from collections import defaultdict
+from typing import Any, TypedDict, Union
+
+import numpy as np
+import numpy.typing as npt
+from gymnasium import spaces
+from playwright.sync_api import CDPSession, Page, ViewportSize
+
+from browser_env.constants import (
+ ASCII_CHARSET,
+ FREQ_UNICODE_CHARSET,
+ IGNORED_ACTREE_PROPERTIES,
+ UTTERANCE_MAX_LENGTH,
+)
+
+from .utils import (
+ AccessibilityTree,
+ AccessibilityTreeNode,
+ BrowserConfig,
+ BrowserInfo,
+ DOMNode,
+ DOMTree,
+ Observation,
+ png_bytes_to_numpy,
+)
+
+from .html_tools import HtmlParser, basic_attrs, print_html_object
+
+IN_VIEWPORT_RATIO_THRESHOLD = 0.8
+
+class TreeNode:
+ def __init__(self, node_id, role, name, depth, **kwargs):
+ self.visible = True
+ self.node_id = node_id
+ self.role = role
+ self.name = name
+ self.depth = depth
+ self.properties = None
+ if "properties" in kwargs.keys():
+ self.properties = kwargs["properties"]
+
+ self.children = []
+ self.parent = None
+
+ def add_child(self, child):
+ child.parent = self
+ self.children.append(child)
+
+ def copy(self):
+ from copy import deepcopy
+ new_self = deepcopy(self)
+ new_self.children = []
+ new_self.parent = None
+ return new_self
+
+ def get_visible_node_number(self):
+ visible_ids = []
+
+ def dfs(current_node):
+ if current_node.visible:
+ visible_ids.append(current_node.node_id)
+ for child in current_node.children:
+ dfs(child)
+
+ dfs(self)
+
+ return len(visible_ids)
+
+ def delete_tree(self):
+ for child in self.children:
+ child.delete_tree()
+ self.children.clear()
+ self.parent = None
+
+ def has_properties(self):
+ return getattr(self, "properties", {})
+
+ def visible_children(self):
+ return [c for c in self.children if c.visible]
+
+ def visible_siblings(self):
+ if not self.parent:
+ return []
+ return [n for n in self.parent.children if n.visible and n.node_id != self.node_id]
+
+ def siblings(self):
+ if not self.parent:
+ return []
+ return [n for n in self.parent.children if n.node_id != self.node_id]
+
+ def search_node_by_id(self, target_id):
+ if self.node_id == target_id or (self.name and f"[{target_id}]" in self.name):
+ return self
+ for child in self.children:
+ result = child.search_node_by_id(target_id)
+ if result:
+ return result
+ return None
+
+ def all_children_invisible(self):
+ if not self.children:
+ return True
+ for child in self.children:
+ if child.visible:
+ return False
+ return True
+
+ def has_the_same_properties_as(self, another_node):
+ node_a_has_properties = getattr(self, "properties", "")
+ node_b_has_properties = getattr(another_node, "properties", "")
+ if not node_a_has_properties and not node_b_has_properties:
+ return True
+ elif (node_a_has_properties and not node_b_has_properties) or (not node_a_has_properties and node_b_has_properties):
+ return False
+ else:
+ return self.properties == another_node.properties
+
+ def is_identical_to(self, another_node):
+ if another_node.children:
+ return False
+ return self.role == another_node.role and self.name == another_node.name and self.has_the_same_properties_as(another_node=another_node)
+
+ def last_sibling(self, visible_required=False):
+ if not self.parent:
+ return None
+ last_sibling_idx = self.parent.children.index(self) - 1
+ if last_sibling_idx < 0:
+ return None
+ if not visible_required:
+ return self.parent.children[last_sibling_idx]
+ for sibling in self.parent.children[:self.parent.children.index(self):-1]:
+ if sibling.visible:
+ return sibling
+ return None
+
+ def next_sibling(self, visible_required=False):
+ if not self.parent:
+ return None
+ next_sibling_idx = self.parent.children.index(self) + 1
+ if next_sibling_idx >= len(self.parent.children):
+ return None
+ if not visible_required:
+ return self.parent.children[next_sibling_idx]
+ for sibling in self.parent.children[next_sibling_idx:]:
+ if sibling.visible:
+ return sibling
+ return None
+
+ def has_identical_siblings(self):
+ if not (self.parent and self.all_children_invisible()):
+ return False
+ if any(sibling.role == self.role and sibling.name == self.name for sibling in self.parent.children if (sibling.node_id != self.node_id and sibling.all_children_invisible())):
+ return True
+ return False
+
+ def has_identical_surrounding_siblings(self):
+ if self.last_sibling(visible_required=False):
+ if self.is_identical_to(self.last_sibling(visible_required=False)):
+ return True
+ if self.last_sibling(visible_required=True):
+ if self.is_identical_to(self.last_sibling(visible_required=True)):
+ return True
+ if self.next_sibling(visible_required=False):
+ if self.is_identical_to(self.next_sibling(visible_required=False)):
+ return True
+ if self.next_sibling(visible_required=True):
+ if self.is_identical_to(self.next_sibling(visible_required=True)):
+ return True
+ return False
+
+ def is_differentiable(self, strict=False):
+ if self.parent and self.parent.role == "row":
+ return True
+ if not strict and self.has_identical_siblings():
+ return False
+ if self.has_identical_surrounding_siblings():
+ return False
+ return True
+
+
+class ObservationProcessor:
+ def process(self, page: Page, client: CDPSession) -> Observation:
+ raise NotImplementedError
+
+
+class ObservationMetadata(TypedDict):
+ obs_nodes_info: dict[str, Any]
+
+
+def create_empty_metadata() -> ObservationMetadata:
+ return {
+ "obs_nodes_info": {},
+ }
+
+
+class TextObervationProcessor(ObservationProcessor):
+ def __init__(
+ self,
+ observation_type: str,
+ current_viewport_only: bool,
+ viewport_size: ViewportSize,
+ ):
+ self.observation_type = observation_type
+ self.current_viewport_only = current_viewport_only
+ self.viewport_size = viewport_size
+ self.observation_tag = "text"
+ self.meta_data = (
+ create_empty_metadata()
+ ) # use the store meta data of this observation type
+
+ def fetch_browser_info(
+ self,
+ page: Page,
+ client: CDPSession,
+ ) -> BrowserInfo:
+ # extract domtree
+ tree = client.send(
+ "DOMSnapshot.captureSnapshot",
+ {
+ "computedStyles": [],
+ "includeDOMRects": True,
+ "includePaintOrder": True,
+ },
+ )
+
+ # calibrate the bounds, in some cases, the bounds are scaled somehow
+ bounds = tree["documents"][0]["layout"]["bounds"]
+ b = bounds[0]
+ n = b[2] / self.viewport_size["width"]
+ bounds = [[x / n for x in bound] for bound in bounds]
+ tree["documents"][0]["layout"]["bounds"] = bounds
+
+ # extract browser info
+ win_top_bound = page.evaluate("window.pageYOffset")
+ win_left_bound = page.evaluate("window.pageXOffset")
+ win_width = page.evaluate("window.screen.width")
+ win_height = page.evaluate("window.screen.height")
+ win_right_bound = win_left_bound + win_width
+ win_lower_bound = win_top_bound + win_height
+ device_pixel_ratio = page.evaluate("window.devicePixelRatio")
+ assert device_pixel_ratio == 1.0, "devicePixelRatio is not 1.0"
+
+ config: BrowserConfig = {
+ "win_top_bound": win_top_bound,
+ "win_left_bound": win_left_bound,
+ "win_width": win_width,
+ "win_height": win_height,
+ "win_right_bound": win_right_bound,
+ "win_lower_bound": win_lower_bound,
+ "device_pixel_ratio": device_pixel_ratio,
+ }
+
+ # assert len(tree['documents']) == 1, "More than one document in the DOM tree"
+ info: BrowserInfo = {"DOMTree": tree, "config": config}
+ # with open('output/browser_info.json', 'w') as f:
+ # f.write(json.dumps(tree, ensure_ascii=False))
+ return info
+
+ @staticmethod
+ def get_bounding_client_rect(
+ client: CDPSession, backend_node_id: str
+ ) -> dict[str, Any]:
+ try:
+ remote_object = client.send(
+ "DOM.resolveNode", {"backendNodeId": int(backend_node_id)}
+ )
+ remote_object_id = remote_object["object"]["objectId"]
+ response = client.send(
+ "Runtime.callFunctionOn",
+ {
+ "objectId": remote_object_id,
+ "functionDeclaration": """
+ function() {
+ if (this.nodeType == 3) {
+ var range = document.createRange();
+ range.selectNode(this);
+ var rect = range.getBoundingClientRect().toJSON();
+ range.detach();
+ return rect;
+ } else {
+ return this.getBoundingClientRect().toJSON();
+ }
+ }
+ """,
+ "returnByValue": True,
+ },
+ )
+ return response
+ except Exception as e:
+ return {"result": {"subtype": "error"}}
+
+ @staticmethod
+ def get_element_in_viewport_ratio(
+ elem_left_bound: float,
+ elem_top_bound: float,
+ width: float,
+ height: float,
+ config: BrowserConfig,
+ ) -> float:
+ elem_right_bound = elem_left_bound + width
+ elem_lower_bound = elem_top_bound + height
+
+ win_left_bound = 0
+ win_right_bound = config["win_width"]
+ win_top_bound = 0
+ win_lower_bound = config["win_height"]
+
+ # Compute the overlap in x and y axes
+ overlap_width = max(
+ 0,
+ min(elem_right_bound, win_right_bound)
+ - max(elem_left_bound, win_left_bound),
+ )
+ overlap_height = max(
+ 0,
+ min(elem_lower_bound, win_lower_bound)
+ - max(elem_top_bound, win_top_bound),
+ )
+
+ # Compute the overlap area
+ ratio = overlap_width * overlap_height / width * height
+ return ratio
+
+ def element_is_visible(self, page, element_id):
+ def _get_element_in_viewport_ratio(
+ elem_left_bound: float,
+ elem_top_bound: float,
+ width: float,
+ height: float,
+ config: BrowserConfig,
+ ) -> float:
+ def calculate_overlap(start1, end1, start2, end2):
+ # Calculate overlap
+ overlap_start = max(start1, start2)
+ overlap_end = min(end1, end2)
+
+ # Check if there's overlap
+ if overlap_start < overlap_end:
+ overlap = overlap_end - overlap_start
+ else:
+ overlap = 0
+
+ return overlap
+ elem_right_bound = elem_left_bound + width
+ elem_lower_bound = elem_top_bound + height
+
+ win_left_bound = 0
+ win_right_bound = config["win_width"]
+ win_top_bound = 0
+ win_lower_bound = config["win_height"]
+
+ overlap_width = calculate_overlap(elem_left_bound, elem_right_bound, win_left_bound, win_right_bound)
+ overlap_height = calculate_overlap(elem_top_bound, elem_lower_bound, win_top_bound, win_lower_bound)
+
+ try:
+ ratio = (overlap_width * overlap_height) / (width * height)
+ return ratio
+ except:
+ return 1 #TODO
+ try:
+ browser_info = self.fetch_browser_info(page, page.client)
+ except Exception:
+ page.wait_for_load_state("load", timeout=500)
+ browser_info = self.fetch_browser_info(page, page.client)
+
+ response = self.get_bounding_client_rect(
+ page.client, self.obs_nodes_info[element_id]["backend_id"]
+ )
+
+ x = response["result"]["value"]["x"]
+ y = response["result"]["value"]["y"]
+ width = response["result"]["value"]["width"]
+ height = response["result"]["value"]["height"]
+
+
+ in_viewport_ratio = _get_element_in_viewport_ratio(
+ elem_left_bound=float(x),
+ elem_top_bound=float(y),
+ width=float(width),
+ height=float(height),
+ config=browser_info["config"],
+ )
+
+ if in_viewport_ratio < IN_VIEWPORT_RATIO_THRESHOLD:
+ return False
+
+ return True
+
+ def fetch_page_html(
+ self,
+ info: BrowserInfo,
+ page: Page,
+ client: CDPSession,
+ current_viewport_only: bool,
+ ) -> DOMTree:
+ # adopted from [natbot](https://github.com/nat/natbot)
+ tree = info["DOMTree"]
+ config = info["config"]
+ strings = tree["strings"]
+ document = tree["documents"][0]
+ nodes = document["nodes"]
+ layout = document["layout"]
+
+ import time
+ stt = time.time()
+ # make a dom tree that is easier to navigate
+ dom_tree: DOMTree = []
+ graph = defaultdict(list)
+ print(nodes.keys())
+ for node_idx in range(len(nodes["nodeName"])):
+ cur_node: DOMNode = {
+ "nodeId": "",
+ "nodeType": "",
+ "nodeName": "",
+ "nodeValue": "",
+ "attributes": "",
+ "backendNodeId": "",
+ "parentId": "",
+ "childIds": [],
+ "cursor": 0,
+ "union_bound": None,
+ }
+
+ node_type_idx = nodes["nodeType"][node_idx]
+ node_type = "generic"
+ if node_type_idx >= 0 and node_type_idx < len(strings):
+ node_type = strings[node_type_idx]
+
+ node_name = strings[nodes["nodeName"][node_idx]]
+
+ node_value_idx = nodes["nodeValue"][node_idx]
+ node_value = ""
+ if node_value_idx >= 0 and node_value_idx < len(strings):
+ node_value = " ".join(strings[node_value_idx].split())
+
+ node_attributes = [
+ strings[i] for i in nodes["attributes"][node_idx]
+ ]
+ node_attributes_str = ""
+ for i in range(0, len(node_attributes), 2):
+ a = node_attributes[i]
+ b = node_attributes[i + 1]
+ # b = " ".join(b.split())
+ import re
+ b = re.sub(r"{\s*opacity:\s*.*;*\s*}", " ", b)
+ b = [b_item for b_item in b.split() if b_item.count('vimium') == 0]
+ b = " ".join(b)
+ node_attributes_str += f'{a}="{b}" '
+
+ node_attributes_str = node_attributes_str.strip()
+
+ cur_node["nodeId"] = str(node_idx)
+ cur_node["nodeType"] = node_type
+ cur_node["nodeName"] = node_name
+ cur_node["nodeValue"] = node_value
+ cur_node["attributes"] = node_attributes_str
+ cur_node["backendNodeId"] = str(nodes["backendNodeId"][node_idx])
+ cur_node["parentId"] = str(nodes["parentIndex"][node_idx])
+
+ if cur_node["parentId"] != "-1":
+ graph[cur_node["parentId"]].append(str(cur_node["nodeId"]))
+
+ # get the bound
+ if cur_node["parentId"] == "-1":
+ cur_node["union_bound"] = [0.0, 0.0, 10.0, 10.0]
+ else:
+ # method 1
+ # response = self.get_bounding_client_rect(
+ # client, cur_node["backendNodeId"]
+ # )
+
+ # if response.get("result", {}).get("subtype", "") == "error":
+ # cur_node["union_bound"] = None
+ # else:
+ # x = response["result"]["value"]["x"]
+ # y = response["result"]["value"]["y"]
+ # width = response["result"]["value"]["width"]
+ # height = response["result"]["value"]["height"]
+ # cur_node["union_bound"] = [x, y, width, height]
+
+ # method 2
+ bound = [0.0, 0.0, 0.0, 0.0]
+ if node_idx in layout["nodeIndex"]:
+ bound = layout["bounds"][layout["nodeIndex"].index(node_idx)]
+ bound[0] -= config["win_left_bound"]
+ bound[1] -= config["win_top_bound"]
+
+ cur_node["union_bound"] = bound
+
+ dom_tree.append(cur_node)
+ print('[build]', time.time() - stt)
+
+ stt = time.time()
+ # add parent children index to the node
+ for parent_id, child_ids in graph.items():
+ dom_tree[int(parent_id)]["childIds"] = child_ids
+ print('[graph]', time.time() - stt)
+
+ # with open('output/dom_tree.json', 'w') as f:
+ # f.write(json.dumps(dom_tree, ensure_ascii=False))
+
+ stt = time.time()
+ # remove the nodes that are not in the current viewport
+ if current_viewport_only:
+
+ def remove_node_in_graph(node: DOMNode) -> None:
+ # update the node information in the accessibility tree
+ node_id = node["nodeId"]
+ parent_id = node["parentId"]
+ child_ids = node["childIds"]
+
+ # update the children of the parent node
+ assert dom_tree[int(parent_id)]["parentId"] != "[REMOVED]"
+ # remove the nodeid from parent
+ index = dom_tree[int(parent_id)]["childIds"].index(node_id)
+ dom_tree[int(parent_id)]["childIds"].pop(index)
+
+ # Insert children_nodeids in the same location
+ for child_id in child_ids:
+ dom_tree[int(parent_id)]["childIds"].insert(
+ index, child_id
+ )
+ index += 1
+
+ # update children node's parent
+ for child_id in child_ids:
+ dom_tree[int(child_id)]["parentId"] = parent_id
+ # mark as removed
+ dom_tree[int(node_id)]["parentId"] = "[REMOVED]"
+
+ config = info["config"]
+ for cursor, node in enumerate(dom_tree):
+ if not node["union_bound"]:
+ remove_node_in_graph(node)
+ continue
+
+ [x, y, width, height] = node["union_bound"]
+
+ # invisible node
+ if width == 0.0 or height == 0.0:
+ parent_id = node["parentId"]
+ if node["nodeName"] not in ['OPTION'] or dom_tree[int(parent_id)]["nodeName"] not in ["SELECT"]:
+ remove_node_in_graph(node)
+ continue
+
+ in_viewport_ratio = self.get_element_in_viewport_ratio(
+ elem_left_bound=float(x),
+ elem_top_bound=float(y),
+ width=float(width),
+ height=float(height),
+ config=config,
+ )
+
+ if in_viewport_ratio < IN_VIEWPORT_RATIO_THRESHOLD:
+ remove_node_in_graph(node)
+
+ dom_tree = [
+ node
+ for node in dom_tree
+ if node.get("parentId", "-1") != "[REMOVED]"
+ ]
+
+ print('[filter]', time.time() - stt)
+ return dom_tree
+
+ @staticmethod
+ def parse_my_html(dom_tree: DOMTree) -> tuple[str, str, dict[str, Any], Any]:
+ """Parse the html tree into a string text"""
+
+ obs_nodes_info = {}
+ nodeid_to_cursor = {
+ node["nodeId"]: idx for idx, node in enumerate(dom_tree)
+ }
+
+ def dfs(node_cursor: int, depth: int) -> tuple[str, list[str]]:
+ tree_str, labeled_elems = '', []
+ node = dom_tree[node_cursor]
+ valid_node = True
+ pure_text = False
+ try:
+ if node['nodeName'] == '#text':
+ node['nodeName'] = 'text'
+
+ node_str = f"<{node['nodeName']}"
+ if node["attributes"]:
+ node_str += f" {node['attributes']}"
+ node_str += f" backend-id=\"bid-{node['backendNodeId']}\"> {node['nodeValue']}"
+
+ # if node['nodeName'] == '#text':
+ # pure_text = True
+ # node_str = node['nodeValue']
+
+ valid_node = bool(node["attributes"] or node["nodeValue"] or pure_text)
+
+ if valid_node:
+ node_html = lxml.html.fromstring(node_str)
+ label = node_html.attrib.get('data-testid', '')
+ if len(label) > 0:
+ labeled_elems.append(node["backendNodeId"])
+ obs_nodes_info[str(node_cursor)] = {
+ "backend_id": node["backendNodeId"],
+ "union_bound": node["union_bound"],
+ "text": node['nodeValue'],
+ "label": label,
+ }
+ tree_str += f"{node_str}"
+
+ except Exception as e:
+ valid_node = False
+
+ for child_ids in node["childIds"]:
+ child_cursor = nodeid_to_cursor[child_ids]
+ child_depth = depth + 1 if valid_node else depth
+ child_str, elems = dfs(child_cursor, child_depth)
+ tree_str += child_str
+ labeled_elems.extend(elems)
+
+ if valid_node and not pure_text:
+ tree_str += f""
+
+ return tree_str, labeled_elems
+
+ html, labeled_elems = dfs(0, 0)
+
+ # with open('output/raw.html', 'w') as f:
+ # f.write(html)
+ print(labeled_elems)
+
+ args = {
+ 'use_position': False,
+ 'id_attr': 'backend-id',
+ 'label_generator': 'order',
+ 'label_attr': 'data-testid',
+ 'attr_list': basic_attrs,
+ 'prompt': 'refine',
+ }
+
+ hp = HtmlParser(html, args)
+ packet = hp.parse_tree()
+ page_html = packet['html']
+
+ print(print_html_object(page_html))
+
+ it, pt = packet.get('init_time', 0), packet.get('parse_time', 0)
+ print(f'[Time] {it:.3f} {pt:.3f}')
+
+ return html, page_html, obs_nodes_info, hp
+
+ @staticmethod
+ def parse_html(dom_tree: DOMTree) -> tuple[str, dict[str, Any]]:
+ """Parse the html tree into a string text"""
+
+ obs_nodes_info = {}
+ nodeid_to_cursor = {
+ node["nodeId"]: idx for idx, node in enumerate(dom_tree)
+ }
+
+ def dfs(node_cursor: int, depth: int) -> str:
+ tree_str = ""
+ node = dom_tree[node_cursor]
+ indent = "\t" * depth
+ valid_node = True
+ try:
+ node_str = f"[{node_cursor}] <{node['nodeName']}"
+ if node["attributes"]:
+ node_str += f" {node['attributes']}"
+ node_str += f"> {node['nodeValue']}"
+ valid_node = bool(node["attributes"] or node["nodeValue"])
+
+ if valid_node:
+ obs_nodes_info[str(node_cursor)] = {
+ "backend_id": node["backendNodeId"],
+ "union_bound": node["union_bound"],
+ "text": node_str,
+ }
+ tree_str += f"{indent}{node_str}\n"
+
+ except Exception as e:
+ valid_node = False
+
+ for child_ids in node["childIds"]:
+ child_cursor = nodeid_to_cursor[child_ids]
+ child_depth = depth + 1 if valid_node else depth
+ child_str = dfs(child_cursor, child_depth)
+ tree_str += child_str
+
+ return tree_str
+
+ html = dfs(0, 0)
+ return html, obs_nodes_info
+
+ def fetch_page_accessibility_tree(
+ self,
+ info: BrowserInfo,
+ client: CDPSession,
+ current_viewport_only: bool,
+ ) -> AccessibilityTree:
+ accessibility_tree: AccessibilityTree = client.send(
+ "Accessibility.getFullAXTree", {}
+ )["nodes"]
+
+ # a few nodes are repeated in the accessibility tree
+ seen_ids = set()
+ _accessibility_tree = []
+ for node in accessibility_tree:
+ if node["nodeId"] not in seen_ids:
+ _accessibility_tree.append(node)
+ seen_ids.add(node["nodeId"])
+ accessibility_tree = _accessibility_tree
+ nodeid_to_cursor = {}
+ for cursor, node in enumerate(accessibility_tree):
+ nodeid_to_cursor[node["nodeId"]] = cursor
+ # usually because the node is not visible etc
+ if "backendDOMNodeId" not in node:
+ node["union_bound"] = None
+ continue
+ backend_node_id = str(node["backendDOMNodeId"])
+ if node["role"]["value"] == "RootWebArea":
+ # always inside the viewport
+ node["union_bound"] = [0.0, 0.0, 10.0, 10.0]
+ else:
+ response = self.get_bounding_client_rect(
+ client, backend_node_id
+ )
+ if response.get("result", {}).get("subtype", "") == "error":
+ node["union_bound"] = None
+ else:
+ x = response["result"]["value"]["x"]
+ y = response["result"]["value"]["y"]
+ width = response["result"]["value"]["width"]
+ height = response["result"]["value"]["height"]
+ node["union_bound"] = [x, y, width, height]
+
+ # filter nodes that are not in the current viewport
+ if current_viewport_only:
+
+ def remove_node_in_graph(node: AccessibilityTreeNode) -> None:
+ # update the node information in the accessibility tree
+ nodeid = node["nodeId"]
+ node_cursor = nodeid_to_cursor[nodeid]
+ parent_nodeid = node["parentId"]
+ children_nodeids = node["childIds"]
+ parent_cursor = nodeid_to_cursor[parent_nodeid]
+ # update the children of the parent node
+ assert (
+ accessibility_tree[parent_cursor].get("parentId", "Root")
+ is not None
+ )
+ # remove the nodeid from parent's childIds
+ index = accessibility_tree[parent_cursor]["childIds"].index(
+ nodeid
+ )
+ accessibility_tree[parent_cursor]["childIds"].pop(index)
+ # Insert children_nodeids in the same location
+ for child_nodeid in children_nodeids:
+ accessibility_tree[parent_cursor]["childIds"].insert(
+ index, child_nodeid
+ )
+ index += 1
+ # update children node's parent
+ for child_nodeid in children_nodeids:
+ child_cursor = nodeid_to_cursor[child_nodeid]
+ accessibility_tree[child_cursor][
+ "parentId"
+ ] = parent_nodeid
+ # mark as removed
+ accessibility_tree[node_cursor]["parentId"] = "[REMOVED]"
+
+ config = info["config"]
+ for node in accessibility_tree:
+ if not node["union_bound"]:
+ remove_node_in_graph(node)
+ continue
+
+ [x, y, width, height] = node["union_bound"]
+
+ # invisible node
+ if width == 0 or height == 0:
+ remove_node_in_graph(node)
+ continue
+
+ in_viewport_ratio = self.get_element_in_viewport_ratio(
+ elem_left_bound=float(x),
+ elem_top_bound=float(y),
+ width=float(width),
+ height=float(height),
+ config=config,
+ )
+
+ if in_viewport_ratio < IN_VIEWPORT_RATIO_THRESHOLD:
+ remove_node_in_graph(node)
+
+ accessibility_tree = [
+ node
+ for node in accessibility_tree
+ if node.get("parentId", "Root") != "[REMOVED]"
+ ]
+
+ return accessibility_tree
+
+ @staticmethod
+ def parse_accessibility_tree(
+ accessibility_tree: AccessibilityTree,
+ ) -> tuple[str, dict[str, Any], TreeNode]:
+ """Parse the accessibility tree into a string text"""
+ node_id_to_idx = {}
+ for idx, node in enumerate(accessibility_tree):
+ node_id_to_idx[node["nodeId"]] = idx
+
+ obs_nodes_info = {}
+
+ def dfs(idx: int, obs_node_id: str, depth: int, active_node_dict: dict) -> str:
+ tree_str = ""
+ node = accessibility_tree[idx]
+ indent = "\t" * depth
+ valid_node = True
+ try:
+ role = node["role"]["value"]
+ name = node["name"]["value"]
+ node_str = f"[{obs_node_id}] {role} {repr(name)}"
+ properties = []
+ structured_properties = {}
+ for property in node.get("properties", []):
+ try:
+ if property["name"] in IGNORED_ACTREE_PROPERTIES:
+ continue
+ properties.append(
+ f'{property["name"]}: {property["value"]["value"]}'
+ )
+ structured_properties[property["name"]] = property["value"]["value"]
+ except KeyError:
+ pass
+
+ if properties:
+ node_str += " " + " ".join(properties)
+
+ # check valid
+ if not node_str.strip():
+ valid_node = False
+
+ # empty generic node
+ if not name.strip():
+ if not properties:
+ if role in [
+ "generic",
+ "img",
+ "list",
+ "strong",
+ "paragraph",
+ "banner",
+ "navigation",
+ "Section",
+ "LabelText",
+ "Legend",
+ "listitem",
+ ]:
+ valid_node = False
+ elif role in ["listitem"]:
+ valid_node = False
+
+ if valid_node:
+ tree_str += f"{indent}{node_str}"
+ obs_nodes_info[obs_node_id] = {
+ "backend_id": node["backendDOMNodeId"],
+ "union_bound": node["union_bound"],
+ "text": node_str,
+ }
+
+ except Exception as e:
+ valid_node = False
+
+ structured_node = TreeNode(node_id=int(obs_node_id), role=node["role"]["value"], name=node["name"]["value"], depth=depth, properties=structured_properties) if valid_node else None
+ active_node_dict[depth] = structured_node if valid_node else active_node_dict.get(depth, None)
+
+ for _, child_node_id in enumerate(node["childIds"]):
+ if child_node_id not in node_id_to_idx:
+ continue
+ # mark this to save some tokens
+ child_depth = depth + 1 if valid_node else depth
+ child_str, child_node = dfs(
+ node_id_to_idx[child_node_id], child_node_id, child_depth, active_node_dict=active_node_dict
+ )
+ if child_str.strip():
+ if tree_str.strip():
+ tree_str += "\n"
+ tree_str += child_str
+ if child_depth > 0 and child_node:
+ active_node_dict[child_depth - 1].add_child(child_node)
+
+ return tree_str, structured_node
+
+ tree_str, structured_node = dfs(0, accessibility_tree[0]["nodeId"], 0, active_node_dict={})
+ return tree_str, obs_nodes_info, structured_node
+
+ @staticmethod
+ def clean_accesibility_tree(tree_str: str) -> str:
+ """further clean accesibility tree"""
+ clean_lines: list[str] = []
+ for line in tree_str.split("\n"):
+ if "statictext" in line.lower():
+ prev_lines = clean_lines[-3:]
+ pattern = r"\[\d+\] StaticText '([^']+)'"
+
+ match = re.search(pattern, line)
+ if match:
+ static_text = match.group(1)
+ if all(
+ static_text not in prev_line
+ for prev_line in prev_lines
+ ):
+ clean_lines.append(line)
+ else:
+ clean_lines.append(line)
+
+ return "\n".join(clean_lines)
+
+ def process(self, page: Page, client: CDPSession, context: str) -> str:
+ # get the tab info
+ open_tabs = page.context.pages
+ # try:
+ # tab_titles = [tab.title() for tab in open_tabs]
+ # current_tab_idx = open_tabs.index(page)
+ # for idx in range(len(open_tabs)):
+ # if idx == current_tab_idx:
+ # tab_titles[
+ # idx
+ # ] = f"Tab {idx} (current): {open_tabs[idx].title()}"
+ # else:
+ # tab_titles[idx] = f"Tab {idx}: {open_tabs[idx].title()}"
+ # tab_title_str = " | ".join(tab_titles)
+ # except Exception:
+ # tab_title_str = " | ".join(
+ # ["Tab {idx}" for idx in range(len(open_tabs))]
+ # )
+
+ try:
+ tab_titles = [tab.title() for tab in open_tabs]
+ current_tab_idx = open_tabs.index(page)
+ for idx in range(len(open_tabs)):
+ if idx == current_tab_idx:
+ tab_titles[
+ idx
+ ] = f"{idx+1}. {open_tabs[idx].title()} <-- current tab"
+ else:
+ tab_titles[idx] = f"{idx+1}. {open_tabs[idx].title()}"
+ tab_title_str = "\n".join(tab_titles)
+ except Exception:
+ tab_title_str = "\n".join(
+ [f"{idx+1}. Default" for idx in range(len(open_tabs))]
+ )
+
+
+ try:
+ browser_info = self.fetch_browser_info(page, client)
+ except Exception:
+ page.wait_for_load_state("load", timeout=500)
+ browser_info = self.fetch_browser_info(page, client)
+
+ if self.observation_type == "html":
+ import time
+ stt = time.time()
+ dom_tree = self.fetch_page_html(
+ browser_info,
+ page,
+ client,
+ current_viewport_only=self.current_viewport_only,
+ )
+
+ print('[fetch]', time.time() - stt)
+
+ stt = time.time()
+ raw_html, content, obs_nodes_info, hp = self.parse_my_html(dom_tree)
+ print('[parse]', time.time() - stt)
+
+ window_height = page.evaluate("window.innerHeight")
+ page_height = page.evaluate('document.documentElement.scrollHeight') / window_height
+ position = page.evaluate("window.scrollY") / window_height
+
+ self.obs_nodes_info = obs_nodes_info
+ self.meta_data["obs_nodes_info"] = obs_nodes_info
+ self.meta_data["position_info"] = {
+ "page_height": page_height,
+ "position": position,
+ }
+ self.meta_data["dom_info"] = {
+ "raw_html": raw_html,
+ "dom_tree": dom_tree,
+ }
+ self.meta_data["html_parser"] = hp
+ self.meta_data["tab_title"] = tab_title_str
+
+ elif self.observation_type == "accessibility_tree":
+ accessibility_tree = self.fetch_page_accessibility_tree(
+ browser_info,
+ client,
+ current_viewport_only=self.current_viewport_only,
+ )
+ content, obs_nodes_info, node_root = self.parse_accessibility_tree(
+ accessibility_tree
+ )
+ content = self.clean_accesibility_tree(content)
+ self.obs_nodes_info = obs_nodes_info
+ page_dialog_message = getattr(page, "dialog_message", "")
+ if page_dialog_message:
+ import copy
+ node_root.properties["page_dialog_message"] = copy.deepcopy(page_dialog_message) + " Retry."
+ page.dialog_message = None
+ self.node_root = node_root
+ self.meta_data["obs_nodes_info"] = obs_nodes_info
+
+ else:
+ raise ValueError(
+ f"Invalid observatrion type: {self.observation_type}"
+ )
+
+ self.browser_config = browser_info["config"]
+ # content = f"{tab_title_str}\n\n{content}"
+ return (content, node_root)
+
+ def get_node_info_by_element_id(self, AXTreeId):
+ return self.node_root.search_node_by_id(AXTreeId)
+
+ def get_element_center(self, element_id: str, page) -> tuple[float, float]:
+ node = self.obs_nodes_info[element_id]
+ backend_node_id = str(node["backend_id"])
+ response = self.get_bounding_client_rect(
+ page.client, backend_node_id
+ )
+ x = response["result"]["value"]["x"]
+ y = response["result"]["value"]["y"]
+ width = response["result"]["value"]["width"]
+ height = response["result"]["value"]["height"]
+ center_x = x + width / 2
+ center_y = y + height / 2
+ return (
+ center_x / self.viewport_size["width"],
+ center_y / self.viewport_size["height"],
+ )
+
+
+class ImageObservationProcessor(ObservationProcessor):
+ def __init__(self, observation_type: str, current_viewport_only: bool):
+ self.observation_type = observation_type
+ self.current_viewport_only = current_viewport_only
+ self.observation_tag = "image"
+ self.meta_data = create_empty_metadata()
+
+ def process(self, page: Page, client: CDPSession, context: str) -> npt.NDArray[np.uint8]:
+ try:
+ screenshot = png_bytes_to_numpy(page.screenshot(full_page=(not self.current_viewport_only)))
+ screenshot = screenshot[:2*screenshot.shape[1], :, :]
+ except:
+ page.wait_for_event("load")
+ screenshot = png_bytes_to_numpy(page.screenshot(full_page=(not self.current_viewport_only)))
+ return screenshot
+
+
+class ObservationHandler:
+ """Main entry point to access all observation processor"""
+
+ def __init__(
+ self,
+ main_observation_type: str,
+ text_observation_type: str,
+ image_observation_type: str,
+ current_viewport_only: bool,
+ viewport_size: ViewportSize,
+ ) -> None:
+ self.main_observation_type = main_observation_type
+ self.text_processor = TextObervationProcessor(
+ text_observation_type, current_viewport_only, viewport_size
+ )
+ self.image_processor = ImageObservationProcessor(
+ image_observation_type, current_viewport_only
+ )
+ self.viewport_size = viewport_size
+
+ def get_observation_space(self) -> spaces.Dict:
+ text_space = spaces.Text(
+ min_length=0,
+ max_length=UTTERANCE_MAX_LENGTH,
+ charset=ASCII_CHARSET + FREQ_UNICODE_CHARSET,
+ )
+
+ image_space = spaces.Box(
+ # Each position stores the RGB values. Note the swapped axes (height first).
+ np.zeros(
+ (self.viewport_size["height"], self.viewport_size["width"], 3),
+ dtype=np.uint8,
+ ),
+ np.ones(
+ (self.viewport_size["height"], self.viewport_size["width"], 3),
+ dtype=np.uint8,
+ )
+ * 255.0,
+ dtype=np.uint8,
+ )
+
+ return spaces.Dict({"text": text_space, "image": image_space})
+
+ def get_observation(
+ self, page: Page, client: CDPSession, context: str = '',
+ ) -> dict[str, Observation]:
+ text_obs = self.text_processor.process(page, client, context)
+ image_obs = self.image_processor.process(page, client, context)
+ return {"text": text_obs, "image": image_obs}
+
+ def get_observation_metadata(self) -> dict[str, ObservationMetadata]:
+ return {
+ "text": self.text_processor.meta_data,
+ "image": self.image_processor.meta_data,
+ }
+
+ @property
+ def action_processor(self) -> ObservationProcessor:
+ """Return the main processor that is associated with the action space"""
+ if self.main_observation_type == "text":
+ return self.text_processor
+ elif self.main_observation_type == "image":
+ return self.image_processor
+ else:
+ raise ValueError("Invalid main observation type")
diff --git a/browser_env/py.typed b/browser_env/py.typed
new file mode 100644
index 0000000..e69de29
diff --git a/browser_env/scripts/__init__.py b/browser_env/scripts/__init__.py
new file mode 100644
index 0000000..719e3e3
--- /dev/null
+++ b/browser_env/scripts/__init__.py
@@ -0,0 +1,43 @@
+import os
+from pathlib import Path
+rootdir = Path(__file__).parent
+
+# marker, gpt-4v-act style
+with open(os.path.join(rootdir, 'local_marker.js'), 'r') as f:
+ local_marker_script = f.read()
+
+with open(os.path.join(rootdir, 'mix_marker.js'), 'r') as f:
+ mix_marker_script = f.read()
+
+with open(os.path.join(rootdir, 'get_data.js'), 'r') as f:
+ get_rect_script = f.read()
+
+# canva handler
+with open(os.path.join(rootdir, 'canva_handler.js'), 'r') as f:
+ canva_handler_script = f.read()
+
+# draw label on page
+with open(os.path.join(rootdir, 'label_marker.js'), 'r') as f:
+ label_marker_script = f.read()
+
+# get text from page
+with open(os.path.join(rootdir, 'get_text.js'), 'r') as f:
+ get_text_script = f.read()
+
+# remove label draw on page
+remove_label_mark_script = """
+ () => {
+ document.querySelectorAll(".our-dom-marker").forEach(item => {
+ document.body.removeChild(item);
+ });
+ }
+"""
+
+remove_id_script = """
+ () => {
+ Array.from(document.getElementsByClassName('possible-clickable-element')).forEach((element) => {
+ element.classList.remove('possible-clickable-element');
+ element.removeAttribute('data-testid');
+ });
+ }
+"""
diff --git a/browser_env/scripts/canva_handler.js b/browser_env/scripts/canva_handler.js
new file mode 100644
index 0000000..997ec78
--- /dev/null
+++ b/browser_env/scripts/canva_handler.js
@@ -0,0 +1,44 @@
+() => {
+ var items = Array.prototype.slice.call(
+ document.querySelectorAll("canvas")
+ );
+
+ var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+ var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+ items = items.map(element => {
+ // var img = element.toDataURL("image/png");
+ var bb = element.getClientRects();
+ var rect = {
+ left: -1,
+ top: -1,
+ right: -1,
+ bottom: -1,
+ width: 0,
+ height: 0,
+ };
+ if (bb.length > 0) {
+ bb = bb[0];
+ rect = {
+ left: Math.max(0, bb.left),
+ top: Math.max(0, bb.top),
+ right: Math.min(vw, bb.right),
+ bottom: Math.min(vh, bb.bottom)
+ };
+ rect = {
+ ...rect,
+ width: rect.right - rect.left,
+ height: rect.bottom - rect.top
+ };
+ }
+
+ return {
+ rects: rect,
+ tag: element.tagName.toLowerCase?.() || "",
+ text: element.textContent.trim().replace(/\s{2,}/g, ' '),
+ // img: img
+ };
+ });
+
+ return items;
+}
\ No newline at end of file
diff --git a/browser_env/scripts/get_data.js b/browser_env/scripts/get_data.js
new file mode 100644
index 0000000..492a52a
--- /dev/null
+++ b/browser_env/scripts/get_data.js
@@ -0,0 +1,73 @@
+(packet) => {
+ function int2str(index) {
+ var str = "";
+ while (index >= 0) {
+ str = String.fromCharCode(65 + index % 26) + str;
+ index = Math.floor(index / 26) - 1;
+ }
+ return str;
+ };
+
+ selector = packet.selector
+ index = packet.startIndex
+ var items = Array.prototype.slice.call(
+ document.querySelectorAll(selector)
+ );
+
+ var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+ var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+ items = items.filter(
+ x => !items.some(y => x.contains(y) && !(x == y))
+ ).map(element => {
+ var bb = element.getClientRects();
+ var rect = {
+ left: 0,
+ top: 0,
+ right: 0,
+ bottom: 0,
+ width: 0,
+ height: 0
+ };
+ var keep = false;
+ var text = "", id = -1;
+ if (bb.length > 0) {
+ bb = bb[0];
+ rect = {
+ left: Math.max(0, bb.left),
+ top: Math.max(0, bb.top),
+ right: Math.min(vw, bb.right),
+ bottom: Math.min(vh, bb.bottom)
+ };
+ rect = {
+ ...rect,
+ width: rect.right - rect.left,
+ height: rect.bottom - rect.top
+ };
+ if (rect.width > 0 || rect.height > 0) {
+ keep = true;
+ if (index >= 0) {
+ id = int2str(index++);
+ element.setAttribute("data-testid", id);
+ }
+ var childNodes = element.childNodes;
+
+ for (var i = 0; i < childNodes.length; i++) {
+ if (childNodes[i].nodeType == Node.TEXT_NODE) {
+ text += childNodes[i].textContent;
+ }
+ }
+ }
+ }
+
+ return {
+ keep: true,
+ id,
+ rects: rect,
+ tag: element.tagName.toLowerCase?.() || "",
+ text,//: element.innerText?.trim().replace(/\s{2,}/g, " ") || ""
+ };
+ }).filter(x => x.keep);
+
+ return [items, index];
+}
\ No newline at end of file
diff --git a/browser_env/scripts/get_text.js b/browser_env/scripts/get_text.js
new file mode 100644
index 0000000..e583ed4
--- /dev/null
+++ b/browser_env/scripts/get_text.js
@@ -0,0 +1,48 @@
+() => {
+ var items = Array.prototype.slice.call(
+ document.querySelectorAll("*")
+ );
+
+ var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+ var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+ const ignoreTags = ["script", "html"];
+ items = items.map(element => {
+ const tag = element.tagName.toLowerCase?.() || "";
+ var bb = element.getClientRects();
+ var keep = false;
+ var text = '';
+
+ const domId = element.getAttribute('data-testid');
+ var id = domId? parseInt(domId): "-";
+
+ if (bb.length > 0) {
+ bb = bb[0];
+ var width = Math.min(vw, bb.right) - Math.max(0, bb.left);
+ var height = Math.min(vh, bb.bottom) - Math.max(0, bb.top);
+
+ if (width > 0 && height > 0) {
+ keep = true;
+ var childNodes = element.childNodes;
+
+ for (var i = 0; i < childNodes.length; i++) {
+ if (childNodes[i].nodeType == Node.TEXT_NODE) {
+ text += childNodes[i].textContent;
+ }
+ }
+ }
+ }
+
+ text = text.trim().replace(/\s{2,}/g, ' ');
+ if (ignoreTags.includes(tag)) keep = false;
+ if (id == "-" && text.length == 0) keep = false;
+
+ return {
+ keep,
+ tag,
+ id,
+ text, //:element.innerText?.trim().replace(/\s{2,}/g, " ") || ""
+ };
+ }).filter(x => x.keep);
+
+ return items;
+}
\ No newline at end of file
diff --git a/browser_env/scripts/label_marker.js b/browser_env/scripts/label_marker.js
new file mode 100644
index 0000000..6c55af5
--- /dev/null
+++ b/browser_env/scripts/label_marker.js
@@ -0,0 +1,65 @@
+(items) => {
+ function getRandomColor() {
+ var letters = '0123456789ABCDEF';
+ var color = '#';
+ for (var i = 0; i < 6; i++) {
+ color += letters[Math.floor(Math.random() * 16)];
+ }
+ return color;
+ }
+
+ items.filter(
+ item => item.id != ""
+ ).forEach((item) => {
+ const bbox = item.rects;
+ const id_string = `dom-marker-id-${index}`;
+
+ index = item.id;
+
+ outerElement = document.createElement("div");
+ outerElement.classList.add("our-dom-marker");
+ // var borderColor = getRandomColor();
+ var borderColor = "#FFFF00";
+ outerElement.style.outline = `2px dashed ${borderColor}`;
+ outerElement.style.position = "fixed";
+ outerElement.style.left = bbox.left - 2 + "px";
+ outerElement.style.top = bbox.top - 2 + "px";
+ outerElement.style.width = bbox.width + 4 + "px";
+ outerElement.style.height = bbox.height + 4 + "px";
+ outerElement.style.pointerEvents = "none";
+ outerElement.style.boxSizing = "border-box";
+ outerElement.style.zIndex = 2147483647;
+
+ innerElement = document.createElement("div");
+ innerElement.classList.add("our-dom-marker");
+ innerElement.style.outline = `2px dashed #222288`;
+ innerElement.style.position = "fixed";
+ innerElement.style.left = bbox.left + "px";
+ innerElement.style.top = bbox.top + "px";
+ innerElement.style.width = bbox.width + "px";
+ innerElement.style.height = bbox.height + "px";
+ innerElement.style.pointerEvents = "none";
+ innerElement.style.boxSizing = "border-box";
+ innerElement.style.zIndex = 2147483647;
+
+ // Add floating label at the corner
+ var label = document.createElement("span");
+ var topPosition = 25;
+ if (bbox.top < 25) topPosition = bbox.top;
+ label.textContent = index;
+ label.style.position = "absolute";
+ label.style.top = `-${topPosition}px`;
+ label.style.left = "0px";
+ label.style.background = borderColor;
+ label.style.color = "black";
+ label.style.padding = "2px 4px";
+ label.style.fontSize = "16px";
+ label.style.borderRadius = "2px";
+ label.style.fontWeight = "bold";
+ outerElement.appendChild(label);
+
+ document.body.appendChild(outerElement);
+ document.body.appendChild(innerElement);
+ })
+ return items;
+}
\ No newline at end of file
diff --git a/browser_env/scripts/local_marker.js b/browser_env/scripts/local_marker.js
new file mode 100644
index 0000000..a21bc0c
--- /dev/null
+++ b/browser_env/scripts/local_marker.js
@@ -0,0 +1,46 @@
+() => {
+ var items = Array.prototype.slice.call(
+ document.querySelectorAll('*')
+ ).map((element) => {
+ var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+ var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+ var rects = [...element.getClientRects()].filter(bb => {
+ var center_x = bb.left + bb.width / 2;
+ var center_y = bb.top + bb.height / 2;
+ var elAtCenter = document.elementFromPoint(center_x, center_y);
+
+ return elAtCenter === element || element.contains(elAtCenter)
+ }).map(bb => {
+ const rect = {
+ left: Math.max(0, bb.left),
+ top: Math.max(0, bb.top),
+ right: Math.min(vw, bb.right),
+ bottom: Math.min(vh, bb.bottom)
+ };
+ return {
+ ...rect,
+ width: rect.right - rect.left,
+ height: rect.bottom - rect.top
+ }
+ });
+
+ var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
+
+ return {
+ element: element,
+ include: window.getComputedStyle(element).cursor == "pointer",
+ area,
+ rects,
+ text: element.textContent.trim().replace(/\s{2,}/g, ' '),
+ };
+ }).filter(item =>
+ item.include && (item.area >= 20)
+ )
+
+ items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
+
+ items.forEach(item => {
+ item.element.classList.add('possible-clickable-element');
+ });
+}
\ No newline at end of file
diff --git a/browser_env/scripts/mix_marker.js b/browser_env/scripts/mix_marker.js
new file mode 100644
index 0000000..b3d02e7
--- /dev/null
+++ b/browser_env/scripts/mix_marker.js
@@ -0,0 +1,148 @@
+() => {
+ var items = Array.prototype.slice.call(
+ document.querySelectorAll('*')
+ ).map(function(element) {
+ var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+ var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+ var rects = [...element.getClientRects()].filter(bb => {
+ var center_x = bb.left + bb.width / 2;
+ var center_y = bb.top + bb.height / 2;
+ var elAtCenter = document.elementFromPoint(center_x, center_y);
+
+ if (!elAtCenter) return false;
+ return elAtCenter === element || element.contains(elAtCenter)
+ }).map(bb => {
+ const rect = {
+ left: Math.max(0, bb.left),
+ top: Math.max(0, bb.top),
+ right: Math.min(vw, bb.right),
+ bottom: Math.min(vh, bb.bottom)
+ };
+ return {
+ ...rect,
+ width: rect.right - rect.left,
+ height: rect.bottom - rect.top
+ }
+ });
+ // var rects = [];
+ var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
+
+ const tagName = element.tagName.toLowerCase?.() || "";
+ let isClickable = ((element.onclick != null) || window.getComputedStyle(element).cursor == "pointer");
+
+ // Insert area elements that provide click functionality to an img.
+ if (tagName === "img") {
+ let mapName = element.getAttribute("usemap");
+ if (mapName) {
+ const imgClientRects = element.getClientRects();
+ mapName = mapName.replace(/^#/, "").replace('"', '\\"');
+ const map = document.querySelector(`map[name=\"${mapName}\"]`);
+ if (map && (imgClientRects.length > 0)) isClickable = true;
+ }
+ }
+
+ if (!isClickable) {
+ const role = element.getAttribute("role");
+ const clickableRoles = [
+ "button",
+ "tab",
+ "link",
+ "checkbox",
+ "menuitem",
+ "menuitemcheckbox",
+ "menuitemradio",
+ "radio",
+ ];
+ if (role != null && clickableRoles.includes(role.toLowerCase())) {
+ isClickable = true;
+ } else {
+ const contentEditable = element.getAttribute("contentEditable");
+ if (
+ contentEditable != null &&
+ ["", "contenteditable", "true"].includes(contentEditable.toLowerCase())
+ ) {
+ isClickable = true;
+ }
+ }
+ }
+
+ // Check for jsaction event listeners on the element.
+ if (!isClickable && element.hasAttribute("jsaction")) {
+ const jsactionRules = element.getAttribute("jsaction").split(";");
+ for (let jsactionRule of jsactionRules) {
+ const ruleSplit = jsactionRule.trim().split(":");
+ if ((ruleSplit.length >= 1) && (ruleSplit.length <= 2)) {
+ const [eventType, namespace, actionName] = ruleSplit.length === 1
+ ? ["click", ...ruleSplit[0].trim().split("."), "_"]
+ : [ruleSplit[0], ...ruleSplit[1].trim().split("."), "_"];
+ if (!isClickable) {
+ isClickable = (eventType === "click") && (namespace !== "none") && (actionName !== "_");
+ }
+ }
+ }
+ }
+
+ if (!isClickable) {
+ const clickableTags = [
+ "input",
+ "textarea",
+ "select",
+ "button",
+ "a",
+ "iframe",
+ "video",
+ "object",
+ "embed",
+ "details"
+ ];
+ isClickable = clickableTags.includes(tagName);
+ }
+
+ if (!isClickable) {
+ if (tagName === "label")
+ isClickable = (element.control != null) && !element.control.disabled;
+ else if (tagName === "img")
+ isClickable = ["zoom-in", "zoom-out"].includes(element.style.cursor);
+ }
+
+ // An element with a class name containing the text "button" might be clickable. However, real
+ // clickables are often wrapped in elements with such class names. So, when we find clickables
+ // based only on their class name, we mark them as unreliable.
+ const className = element.getAttribute("class");
+ if (!isClickable && className && className.toLowerCase().includes("button")) {
+ isClickable = true;
+ }
+
+ // Elements with tabindex are sometimes useful, but usually not. We can treat them as second
+ // class citizens when it improves UX, so take special note of them.
+ const tabIndexValue = element.getAttribute("tabindex");
+ const tabIndex = tabIndexValue ? parseInt(tabIndexValue) : -1;
+ if (!isClickable && !(tabIndex < 0) && !isNaN(tabIndex)) {
+ isClickable = true;
+ }
+
+ const idValue = element.getAttribute("id");
+ const id = idValue ? idValue.toLowerCase() : "";
+ if (isClickable && area == 0) {
+ const textValue = element.textContent.trim().replace(/\s{2,}/g, ' ');
+ clickable_msg = `${tagName}[id=${id}] ${isClickable} (${area}) ${textValue}`
+ }
+
+ return {
+ element: element,
+ include: isClickable,
+ area,
+ rects,
+ text: element.textContent.trim().replace(/\s{2,}/g, ' ')
+ };
+ }).filter(item =>
+ item.include && (item.area >= 1)
+ );
+
+ items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
+
+ items.forEach(item => {
+ item.element.classList.add('possible-clickable-element');
+ });
+}
\ No newline at end of file
diff --git a/browser_env/trajectory.py b/browser_env/trajectory.py
new file mode 100644
index 0000000..1c4c410
--- /dev/null
+++ b/browser_env/trajectory.py
@@ -0,0 +1,6 @@
+from typing import Union
+
+from .actions import Action
+from .utils import StateInfo
+
+Trajectory = list[Union[StateInfo, Action]]
diff --git a/browser_env/utils.py b/browser_env/utils.py
new file mode 100644
index 0000000..1814242
--- /dev/null
+++ b/browser_env/utils.py
@@ -0,0 +1,80 @@
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Any, Dict, TypedDict, Union
+
+import numpy as np
+import numpy.typing as npt
+from PIL import Image
+
+
+@dataclass
+class DetachedPage:
+ url: str
+ content: str # html
+
+
+def png_bytes_to_numpy(png: bytes) -> npt.NDArray[np.uint8]:
+ """Convert png bytes to numpy array
+
+ Example:
+
+ >>> fig = go.Figure(go.Scatter(x=[1], y=[1]))
+ >>> plt.imshow(png_bytes_to_numpy(fig.to_image('png')))
+ """
+ return np.array(Image.open(BytesIO(png)))
+
+
+class AccessibilityTreeNode(TypedDict):
+ nodeId: str
+ ignored: bool
+ role: dict[str, Any]
+ chromeRole: dict[str, Any]
+ name: dict[str, Any]
+ properties: list[dict[str, Any]]
+ childIds: list[str]
+ parentId: str
+ backendDOMNodeId: str
+ frameId: str
+ bound: list[float] | None
+ union_bound: list[float] | None
+ offsetrect_bound: list[float] | None
+
+
+class DOMNode(TypedDict):
+ nodeId: str
+ nodeType: str
+ nodeName: str
+ nodeValue: str
+ attributes: str
+ backendNodeId: str
+ parentId: str
+ childIds: list[str]
+ cursor: int
+ union_bound: list[float] | None
+
+
+class BrowserConfig(TypedDict):
+ win_top_bound: float
+ win_left_bound: float
+ win_width: float
+ win_height: float
+ win_right_bound: float
+ win_lower_bound: float
+ device_pixel_ratio: float
+
+
+class BrowserInfo(TypedDict):
+ DOMTree: dict[str, Any]
+ config: BrowserConfig
+
+
+AccessibilityTree = list[AccessibilityTreeNode]
+DOMTree = list[DOMNode]
+
+
+Observation = str | npt.NDArray[np.uint8]
+
+
+class StateInfo(TypedDict):
+ observation: dict[str, Observation]
+ info: Dict[str, Any]
diff --git a/config_files/tasks/stanford_cs_head.json b/config_files/tasks/stanford_cs_head.json
new file mode 100644
index 0000000..15f1de2
--- /dev/null
+++ b/config_files/tasks/stanford_cs_head.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google"
+ ],
+ "task_id": "stanford_cs_head",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "What is the email address of the Dean of the School of Engineering at Stanford University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "widom@stanford.edu"
+ }
+ }
+ }
\ No newline at end of file
diff --git a/config_files/tasks/tsinghua_da_head.json b/config_files/tasks/tsinghua_da_head.json
new file mode 100644
index 0000000..19ea048
--- /dev/null
+++ b/config_files/tasks/tsinghua_da_head.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google"
+ ],
+ "task_id": "tsinghua_da_head",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "帮我查一下清华大学自动化系系主任的邮箱",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "taozhang@tsinghua.edu.cn"
+ }
+ }
+ }
\ No newline at end of file
diff --git a/config_files/test.raw.json b/config_files/test.raw.json
new file mode 100644
index 0000000..fe76d23
--- /dev/null
+++ b/config_files/test.raw.json
@@ -0,0 +1,29623 @@
+[
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 0,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the top-{{n}} best-selling product in {{year}}",
+ "instantiation_dict": {
+ "n": 1,
+ "year": 2022
+ },
+ "intent": "What is the top-1 best-selling product in 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Quest Lumaflex\u2122 Band"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 1,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the top-{{n}} best-selling brand in {{period}}",
+ "instantiation_dict": {
+ "n": 1,
+ "period": "Quarter 1 2022"
+ },
+ "intent": "What is the top-1 best-selling brand in Quarter 1 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Sprite"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Sprite"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 2,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
+ "instantiation_dict": {
+ "n": 1,
+ "period": "Quarter 1 2022"
+ },
+ "intent": "What is the top-1 best-selling product type in Quarter 1 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Yoga ball"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yoga ball"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 3,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the top-{{n}} best-selling product in {{year}}",
+ "instantiation_dict": {
+ "n": 2,
+ "year": 2022
+ },
+ "intent": "What are the top-2 best-selling product in 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Quest Lumaflex\u2122 Band",
+ "Sprite Stasis Ball 65 cm"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Quest Lumaflex\u2122 Band, Sprite Stasis Ball 65 cm"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 4,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the top-{{n}} best-selling product in {{period}}",
+ "instantiation_dict": {
+ "n": 3,
+ "period": "Jan 2023"
+ },
+ "intent": "What are the top-3 best-selling product in Jan 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Impulse Duffle",
+ "Overnight Duffle",
+ "Hawkeye Yoga Short-32-Blue"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Impulse Duffle, Overnight Duffle, Hawkeye Yoga Short-32-Blue"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 5,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the top-{{n}} best-selling product type in {{period}}",
+ "instantiation_dict": {
+ "n": 1,
+ "period": "Jan 2023"
+ },
+ "intent": "What is the top-1 best-selling product type in Jan 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Duffle"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Duffle"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 6,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the top-{{n}} best-selling product in {{year}}",
+ "instantiation_dict": {
+ "n": 5,
+ "year": 2023
+ },
+ "intent": "What are the top-5 best-selling product in 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Sprite Yoga Strap 6 foot",
+ "Overnight Duffle",
+ "Ida Workout Parachute Pant-29-Purple",
+ "Hawkeye Yoga Short-32-Blue",
+ "Sprite Stasis Ball 65 cm"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Sprite Yoga Strap 6 foot, Overnight Duffle, Ida Workout Parachute Pant-29-Purple, Hawkeye Yoga Short-32-Blue, Sprite Stasis Ball 65 cm"
+ },
+ "intent_template_id": 279
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 7,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
+ "instantiation_dict": {
+ "airport_type": "international airports",
+ "start": "Carnegie Mellon University",
+ "radius": "50 km"
+ },
+ "intent": "Tell me the full address of all international airports that are within a driving distance of 50 km to Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
+ },
+ "intent_template_id": 79
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 8,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
+ "instantiation_dict": {
+ "airport_type": "international airports",
+ "start": "Carnegie Mellon University",
+ "radius": "5 km"
+ },
+ "intent": "Tell me the full address of all international airports that are within a driving distance of 5 km to Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "There is no airport within 5 km of Carnegie Mellon University"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "There is no airport within 5 km of Carnegie Mellon University"
+ },
+ "intent_template_id": 79
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 9,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
+ "instantiation_dict": {
+ "airport_type": "international airports",
+ "start": "Carnegie Art Museum",
+ "radius": "30 km"
+ },
+ "intent": "Tell me the full address of all international airports that are within a driving distance of 30 km to Carnegie Art Museum",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Pittsburgh International Airport, Southern Beltway, Findlay Township, Allegheny County, 15231, United States"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Pittsburgh International Airport People Movers, Airport Boulevard, Findlay Township, Allegheny County, Pennsylvania, 15231, United States"
+ },
+ "intent_template_id": 79
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 10,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the full address of all {{airport_type}} that are within a driving distance of {{radius}} to {{start}}",
+ "instantiation_dict": {
+ "airport_type": "US international airports",
+ "start": "Niagara Falls",
+ "radius": "60 km"
+ },
+ "intent": "Tell me the full address of all US international airports that are within a driving distance of 60 km to Niagara Falls",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States",
+ "Buffalo-Niagara International Airport, Holtz Drive, Town of Cheektowaga, Erie County, New York, 14225, United States"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Niagara Falls International Airport, 2035, Niagara Falls Boulevard, City of Niagara Falls, Town of Wheatfield, Niagara County, New York, 14304, United States Buffalo-Niagara International Airport, South Youngs Road, Town of Cheektowaga, Erie County, New York, 14221, United States"
+ },
+ "intent_template_id": 79
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 11,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
+ "instantiation_dict": {
+ "term": "disappointed"
+ },
+ "intent": "Tell me the the number of reviews that our store received by far that mention term \"disappointed\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "6"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "6"
+ },
+ "intent_template_id": 288
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 12,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
+ "instantiation_dict": {
+ "term": "satisfied"
+ },
+ "intent": "Tell me the the number of reviews that our store received by far that mention term \"satisfied\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2"
+ },
+ "intent_template_id": 288
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 13,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
+ "instantiation_dict": {
+ "term": "decent"
+ },
+ "intent": "Tell me the the number of reviews that our store received by far that mention term \"decent\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2"
+ },
+ "intent_template_id": 288
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 14,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
+ "instantiation_dict": {
+ "term": "not useful"
+ },
+ "intent": "Tell me the the number of reviews that our store received by far that mention term \"not useful\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 288
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 15,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the the number of reviews that our store received by far that mention term \"{{term}}\"",
+ "instantiation_dict": {
+ "term": "best"
+ },
+ "intent": "Tell me the the number of reviews that our store received by far that mention term \"best\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2"
+ },
+ "intent_template_id": 288
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 16,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
+ "instantiation_dict": {
+ "start": "5000 Fifth Avenue, Pittsburgh",
+ "end": "UPMC family health center"
+ },
+ "intent": "Compare the time for walking and driving route from 5000 Fifth Avenue, Pittsburgh to UPMC family health center",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "driving: 2min; walking: 16min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Driving: 2min. Walking: 16min."
+ },
+ "intent_template_id": 73
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 17,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
+ "instantiation_dict": {
+ "start": "AMC Waterfront",
+ "end": "Carnegie Mellon University"
+ },
+ "intent": "Compare the time for walking and driving route from AMC Waterfront to Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "driving: 13min; walking: 1h 35min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "driving: 13min, walking: 1h 35min."
+ },
+ "intent_template_id": 73
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 18,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
+ "instantiation_dict": {
+ "start": "AMC Waterfront",
+ "end": "Univ of Pittsburgh"
+ },
+ "intent": "Compare the time for walking and driving route from AMC Waterfront to Univ of Pittsburgh",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "driving: 15min; walking: 1h 47min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "driving: 15min, walking: 1h 47min."
+ },
+ "intent_template_id": 73
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 19,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Compare the time for walking and driving route from {{start}} to {{end}}",
+ "instantiation_dict": {
+ "start": "Carnegie Science Center",
+ "end": "Carnegie Mellon University"
+ },
+ "intent": "Compare the time for walking and driving route from Carnegie Science Center to Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "driving: 12min; walking: 1h 44min."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "driving: 12min, walking: 1h 44min."
+ },
+ "intent_template_id": 73
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 20,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Compare the difference in time for walking and driving route from {{start}} to {{end}}",
+ "instantiation_dict": {
+ "start": "Randyland",
+ "end": "Carnegie Mellon University"
+ },
+ "intent": "Compare the difference in time for walking and driving route from Randyland to Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "driving: 13min; walking: 1h 45min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "driving: 13min, walking: 1h 45min."
+ },
+ "intent_template_id": 73
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 21,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/6s-wireless-headphones-over-ear-noise-canceling-hi-fi-bass-foldable-stereo-wireless-kid-headsets-earbuds-with-built-in-mic-micro-sd-tf-fm-for-iphone-samsung-ipad-pc-black-gold.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "ear cups being small"
+ },
+ "intent": "List out reviewers, if exist, who mention about ear cups being small",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Joseph Brzezinski",
+ "Catso",
+ "Dibbins",
+ "Anglebert Dinkherhump",
+ "Michelle Davis"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Joseph Brzezinski, Catso, Dibbins, Anglebert Dinkherhump, Michelle Davis"
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 22,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/fujifilm-finepix-z200fd-10mp-digital-camera-with-5x-optical-dual-image-stabilized-zoom-black.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "under water photo"
+ },
+ "intent": "List out reviewers, if exist, who mention about under water photo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no review about under water photo",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 23,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/3-pack-samsung-galaxy-s6-screen-protector-nearpow-tempered-glass-screen-protector-with-9h-hardness-crystal-clear-easy-bubble-free-installation-scratch-resist.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "good fingerprint resistant"
+ },
+ "intent": "List out reviewers, if exist, who mention about good fingerprint resistant",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Rachel",
+ "T. Gannon"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Rachel, T. Gannon, "
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 24,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/haflinger-men-s-wool-felt-open-back-slippers-beige-550-peat-us-7.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "price being unfair"
+ },
+ "intent": "List out reviewers, if exist, who mention about price being unfair",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no reivew about price being unfair",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 25,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "average print quality"
+ },
+ "intent": "List out reviewers, if exist, who mention about average print quality",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Goldfish",
+ "Roxanne Brandon Coffey"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "GoldfishGoldfish, Roxanne Brandon Coffey"
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 26,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/epson-workforce-wf-3620-wifi-direct-all-in-one-color-inkjet-printer-copier-scanner-amazon-dash-replenishment-ready.html",
+ "geolocation": null,
+ "intent_template": "List out reviewers, if exist, who mention about {{description}}",
+ "instantiation_dict": {
+ "description": "complain of the customer service"
+ },
+ "intent": "List out reviewers, if exist, who mention about complain of the customer service",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Bob in Vegas",
+ "RemyR"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Bob in Vegas, RemyRRemyR"
+ },
+ "intent_template_id": 222
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 27,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
+ "instantiation_dict": {
+ "forum": "Showerthoughts"
+ },
+ "intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Showerthoughts forum.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 33
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 28,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
+ "instantiation_dict": {
+ "forum": "Worcester"
+ },
+ "intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the Worcester forum.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 33
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 29,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
+ "instantiation_dict": {
+ "forum": "DIY"
+ },
+ "intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the DIY forum.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 33
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 30,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
+ "instantiation_dict": {
+ "forum": "space"
+ },
+ "intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the space forum.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 33
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 31,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the {{forum}} forum.",
+ "instantiation_dict": {
+ "forum": "photoshopbattles"
+ },
+ "intent": "Tell me the count of comments that have received more downvotes than upvotes for the user who made the latest post on the photoshopbattles forum.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 33
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 32,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
+ "instantiation_dict": {
+ "place": "Pittsburgh Airport",
+ "information": "the walking distance",
+ "target1": "Hilton hotel",
+ "target2": "the nearest supermarket own by a local company"
+ },
+ "intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the walking distance to the nearest supermarket own by a local company from the hotel.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "DoubleTree by Hilton Hotel Pittsburgh Airport",
+ "2.0km"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 2.0km"
+ },
+ "intent_template_id": 78
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 33,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
+ "instantiation_dict": {
+ "place": "Pittsburgh Airport",
+ "target1": "Hilton hotel",
+ "information": "the shortest walking distance",
+ "target2": "a supermarket"
+ },
+ "intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hilton hotel in the vicinity, if available. Then, tell me the the shortest walking distance to a supermarket from the hotel.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "DoubleTree by Hilton Hotel Pittsburgh Airport",
+ "1.4km"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "DoubleTree by Hilton Hotel Pittsburgh Airport Distance: 1.4km"
+ },
+ "intent_template_id": 78
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 34,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
+ "instantiation_dict": {
+ "place": "Pittsburgh Airport",
+ "target1": "Hyatt hotel",
+ "information": "the shortest walking time",
+ "target2": "a supermarket"
+ },
+ "intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the shortest walking time to a supermarket from the hotel.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Hyatt Regency Pittsburgh International Airport"
+ ],
+ "fuzzy_match": [
+ "Time: 3h 30min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport\n3:30"
+ },
+ "intent_template_id": 78
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 35,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I will arrive {{place}} soon. Provide the name of a {{target1}} in the vicinity, if available. Then, tell me the {{information}} to {{target2}} from the hotel.",
+ "instantiation_dict": {
+ "place": "Pittsburgh Airport",
+ "target1": "Hyatt hotel",
+ "information": "the minimal driving time",
+ "target2": "a supermarket"
+ },
+ "intent": "I will arrive Pittsburgh Airport soon. Provide the name of a Hyatt hotel in the vicinity, if available. Then, tell me the the minimal driving time to a supermarket from the hotel.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Hyatt Regency Pittsburgh International Airport"
+ ],
+ "fuzzy_match": [
+ "Time: 15min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Time: 15min"
+ },
+ "intent_template_id": 78
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 36,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
+ "instantiation_dict": {
+ "place": "social security administration",
+ "location": "Carnegie Mellon University"
+ },
+ "intent": "Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Yes"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yes"
+ },
+ "intent_template_id": 77
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 37,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
+ "instantiation_dict": {
+ "place": "police station",
+ "location": "gates building at CMU"
+ },
+ "intent": "Check if the police station in pittsburgh can be reached in one hour by car from gates building at CMU",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Yes"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yes"
+ },
+ "intent_template_id": 77
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 38,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
+ "instantiation_dict": {
+ "place": "duquesne university",
+ "location": "pittsburgh airport"
+ },
+ "intent": "Check if the duquesne university in pittsburgh can be reached in one hour by car from pittsburgh airport",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Yes"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yes"
+ },
+ "intent_template_id": 77
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 39,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
+ "instantiation_dict": {
+ "place": "walmart",
+ "location": "5600 fifth avenue"
+ },
+ "intent": "Check if the walmart in pittsburgh can be reached in one hour by car from 5600 fifth avenue",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Yes"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yes"
+ },
+ "intent_template_id": 77
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 40,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Check if the {{place}} in pittsburgh can be reached in one hour by car from {{location}}",
+ "instantiation_dict": {
+ "place": "amc theatre",
+ "location": "hobart street"
+ },
+ "intent": "Check if the amc theatre in pittsburgh can be reached in one hour by car from hobart street",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Yes"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Yes"
+ },
+ "intent_template_id": 77
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 41,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "List the top {{n}} search terms in my store",
+ "instantiation_dict": {
+ "n": "1"
+ },
+ "intent": "List the top 1 search terms in my store",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "hollister"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "hollister"
+ },
+ "intent_template_id": 285
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 42,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "List the top {{n}} search terms in my store",
+ "instantiation_dict": {
+ "n": "2"
+ },
+ "intent": "List the top 2 search terms in my store",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "hollister",
+ "Joust Bag"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "hollister, Joust Bag"
+ },
+ "intent_template_id": 285
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 43,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "List the top {{n}} search terms in my store",
+ "instantiation_dict": {
+ "n": "3"
+ },
+ "intent": "List the top 3 search terms in my store",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "hollister",
+ "Joust Bag",
+ "Antonia Racer Tank"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "hollister, Joust Bag, Antonia Race Tank"
+ },
+ "intent_template_id": 285
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 44,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Check out my todos",
+ "instantiation_dict": {},
+ "intent": "Check out my todos",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/dashboard/todos",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 303
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 45,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "Check out the most recent open issues",
+ "instantiation_dict": {},
+ "intent": "Check out the most recent open issues",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?sort=created_asc&state=opened",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 300
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 46,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Check out the most recent open issues",
+ "instantiation_dict": {},
+ "intent": "Check out the most recent open issues",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/issues/?sort=created_date&state=opened",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 300
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 47,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
+ "instantiation_dict": {
+ "period": "over the past month"
+ },
+ "intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past month, and the total amount of money I spent.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "0 order",
+ "$0 total spend"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0 order, $0 total spend"
+ },
+ "intent_template_id": 197
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 48,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
+ "instantiation_dict": {
+ "period": "over the past three days"
+ },
+ "intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past three days, and the total amount of money I spent.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "0 order",
+ "$0 total spend"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0 order, $0 total spend"
+ },
+ "intent_template_id": 197
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 49,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
+ "instantiation_dict": {
+ "period": "over the past four month"
+ },
+ "intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past four month, and the total amount of money I spent.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "3 orders",
+ "$845.49 total spend"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3 orders, $845.49 total spend"
+ },
+ "intent_template_id": 197
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 50,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
+ "instantiation_dict": {
+ "period": "over the past year"
+ },
+ "intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past year, and the total amount of money I spent.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "24 orders",
+ "$6560.69 total spend"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "24 orders, $6560.69 total spend"
+ },
+ "intent_template_id": 197
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 51,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Today is 6/12/2023. Tell me how many fulfilled orders I have {{period}}, and the total amount of money I spent.",
+ "instantiation_dict": {
+ "period": "over the past six month"
+ },
+ "intent": "Today is 6/12/2023. Tell me how many fulfilled orders I have over the past six month, and the total amount of money I spent.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "12 orders",
+ "$1603.69 total spend"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "12 orders, $1603.69 total spend"
+ },
+ "intent_template_id": 197
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 52,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How long does it take to walk from {{start}} to {{end}}?",
+ "instantiation_dict": {
+ "start": "Carnegie Mellon University",
+ "end": "starbucks on Craig Street"
+ },
+ "intent": "How long does it take to walk from Carnegie Mellon University to starbucks on Craig Street?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "7 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "7 min"
+ },
+ "intent_template_id": 68
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 53,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How long does it take to walk from {{start}} to {{end}}?",
+ "instantiation_dict": {
+ "start": "Univ of Pittsburgh",
+ "end": "starbucks on Craig Street"
+ },
+ "intent": "How long does it take to walk from Univ of Pittsburgh to starbucks on Craig Street?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "18 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "18 min"
+ },
+ "intent_template_id": 68
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 54,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How long does it take to walk from {{start}} to {{end}}?",
+ "instantiation_dict": {
+ "start": "Carnegie Mellon University",
+ "end": "Univ of Pittsburgh"
+ },
+ "intent": "How long does it take to walk from Carnegie Mellon University to Univ of Pittsburgh?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "25 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "25 min"
+ },
+ "intent_template_id": 68
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 55,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How long does it take to walk from {{start}} to {{end}}?",
+ "instantiation_dict": {
+ "start": "the starbuck near CMU",
+ "end": "Chatham university"
+ },
+ "intent": "How long does it take to walk from the starbuck near CMU to Chatham university?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "30 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "30 min"
+ },
+ "intent_template_id": 68
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 56,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How long does it take to walk from {{start}} to {{end}}?",
+ "instantiation_dict": {
+ "start": "Carnegie Museum of Art",
+ "end": "a library at CMU"
+ },
+ "intent": "How long does it take to walk from Carnegie Museum of Art to a library at CMU?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "11 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "11 min"
+ },
+ "intent_template_id": 68
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 57,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
+ "instantiation_dict": {
+ "place1": "restaurant",
+ "place2": "university center at Carnegie Mellon University"
+ },
+ "intent": "Tell me the closest restaurant(s) to university center at Carnegie Mellon University",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "El Gallo de Oro",
+ "Back Bar Grill",
+ "Grano",
+ "Beefsteak",
+ "Nourish",
+ "Schatz Dining Room",
+ "Au Bon Pain"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "El Gallo de Oro, Back Bar Grill, Grano, Beefsteak, Nourish, Schatz Dining Room, Au Bon Pain"
+ },
+ "intent_template_id": 69
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 58,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
+ "instantiation_dict": {
+ "place1": "cafe",
+ "place2": "CMU Hunt library"
+ },
+ "intent": "Tell me the closest cafe(s) to CMU Hunt library",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "De Fer Coffee & Tea"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "De Fer Coffee & Tea"
+ },
+ "intent_template_id": 69
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 59,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
+ "instantiation_dict": {
+ "place1": "restaurant",
+ "place2": "CMU Hunt library"
+ },
+ "intent": "Tell me the closest restaurant(s) to CMU Hunt library",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "The exchange"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "The exchange"
+ },
+ "intent_template_id": 69
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 60,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
+ "instantiation_dict": {
+ "place1": "restaurant",
+ "place2": "CMU Posner Hall"
+ },
+ "intent": "Tell me the closest restaurant(s) to CMU Posner Hall",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "The exchange"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "The exchange"
+ },
+ "intent_template_id": 69
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 61,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the closest {{place1}}(s) to {{place2}}",
+ "instantiation_dict": {
+ "place1": "restaurant",
+ "place2": "CMU Sorrells Library"
+ },
+ "intent": "Tell me the closest restaurant(s) to CMU Sorrells Library",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "La Prima Espresso"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "La Prima Espresso"
+ },
+ "intent_template_id": 69
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 62,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
+ "instantiation_dict": {
+ "quantifier": "most"
+ },
+ "intent": "Which customer has completed the most number of orders in the entire history?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Jane Smith"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Jane Smith"
+ },
+ "intent_template_id": 276
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 63,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Which customer(s) has completed the {{quantifier}} number of orders in the entire history?",
+ "instantiation_dict": {
+ "quantifier": "second most"
+ },
+ "intent": "Which customer(s) has completed the second most number of orders in the entire history?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Adam Garcia",
+ "Michael Nguyen",
+ "Sarah Miller"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Adam Garcia, Michael Nguyen, Sarah Miller"
+ },
+ "intent_template_id": 276
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 64,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Which customer has placed {{number}} orders in the entire history?",
+ "instantiation_dict": {
+ "number": "2"
+ },
+ "intent": "Which customer has placed 2 orders in the entire history?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Lisa Kim",
+ "Lisa Green",
+ "Julia Williams",
+ "Brian Smith",
+ "Alexander Thomas"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lisa Kim, Lisa Green, Julia Williams, Brian Smith, Alexander Thomas"
+ },
+ "intent_template_id": 276
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 65,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Which customer has completed the {{quantifier}} number of orders in the entire history?",
+ "instantiation_dict": {
+ "quantifier": "fifth most"
+ },
+ "intent": "Which customer has completed the fifth most number of orders in the entire history?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Jane Doe"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Jane Doe"
+ },
+ "intent_template_id": 276
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 66,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
+ "instantiation_dict": {
+ "number": 10,
+ "subreddit": "books",
+ "description": "show me the post URLs that recommand a single book"
+ },
+ "intent": "Among the top 10 post in \"books\" forum, show me the post URLs that recommand a single book",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "http://www.reddit.com/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol",
+ "http://www.reddit.com/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "__REDDIT__/f/books/59396/apple-books-has-a-free-audiobook-of-a-christmas-carol, __REDDIT__/f/books/17445/i-just-finished-reading-the-hobbit-to-my-6-year-old-daughter"
+ },
+ "intent_template_id": 17
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 67,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
+ "instantiation_dict": {
+ "number": 10,
+ "subreddit": "books",
+ "description": "show me the book names from posts that recommand a single book"
+ },
+ "intent": "Among the top 10 post in \"books\" forum, show me the book names from posts that recommand a single book",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "A Christmas Carol",
+ "The Hobbit"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "A Christmas Carol, The Hobbit"
+ },
+ "intent_template_id": 17
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 68,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
+ "instantiation_dict": {
+ "number": 10,
+ "subreddit": "books",
+ "description": "show me the author name and the book name from posts that recommand a single book"
+ },
+ "intent": "Among the top 10 post in \"books\" forum, show me the author name and the book name from posts that recommand a single book",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "A Christmas Carol",
+ "Levar Burton",
+ "The Hobbit",
+ "J. R. R. Tolkien"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "A Christmas Carol by Levar Burton: , The Hobbit by J. R. R. Tolkien"
+ },
+ "intent_template_id": 17
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 69,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Among the top {{number}} post in \"{{subreddit}}\" forum, {{description}}",
+ "instantiation_dict": {
+ "number": 10,
+ "subreddit": "books",
+ "description": "is there any post talks about supporting local book stores? If so, tell me the organizations involved"
+ },
+ "intent": "Among the top 10 post in \"books\" forum, is there any post talks about supporting local book stores? If so, tell me the organizations involved",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "bookshop.org"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "bookshop.org"
+ },
+ "intent_template_id": 17
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 70,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the zip code of {{place}}?",
+ "instantiation_dict": {
+ "place": "Carnegie Mellon University"
+ },
+ "intent": "What is the zip code of Carnegie Mellon University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "15213"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "15213"
+ },
+ "intent_template_id": 70
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 71,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the zip code of {{place}}?",
+ "instantiation_dict": {
+ "place": "Chatham University"
+ },
+ "intent": "What is the zip code of Chatham University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "15232"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "15232"
+ },
+ "intent_template_id": 70
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 72,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the zip code of {{place}}?",
+ "instantiation_dict": {
+ "place": "Yale University"
+ },
+ "intent": "What is the zip code of Yale University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "06516"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "06516"
+ },
+ "intent_template_id": 70
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 73,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the zip code of {{place}}?",
+ "instantiation_dict": {
+ "place": "Columbia University"
+ },
+ "intent": "What is the zip code of Columbia University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "10027"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "10027"
+ },
+ "intent_template_id": 70
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 74,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "instantiation_dict": {
+ "place_list": [
+ "Carnegie Mellon University",
+ "apple store shadyside",
+ "starbucks on craig street"
+ ]
+ },
+ "intent": "Given the following locations, ['Carnegie Mellon University', 'apple store shadyside', 'starbucks on craig street'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "The order is Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Carnegie Mellon University, starbucks on forbes ave, apple store shadyside"
+ },
+ "intent_template_id": 65
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 75,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "instantiation_dict": {
+ "place_list": [
+ "Massachusetts Institute of Technology",
+ "Harvard University",
+ "Boston Logan International Airport"
+ ]
+ },
+ "intent": "Given the following locations, ['Massachusetts Institute of Technology', 'Harvard University', 'Boston Logan International Airport'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "The order is Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Massachusetts Institute of Technology, Harvard University, Boston Logan International Airport"
+ },
+ "intent_template_id": 65
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 76,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Given the following locations, {{place_list}}, what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "instantiation_dict": {
+ "place_list": [
+ "Princeton University",
+ "Yale University",
+ "Harvard University"
+ ]
+ },
+ "intent": "Given the following locations, ['Princeton University', 'Yale University', 'Harvard University'], what would be the optimal route to travel through them all in order to minimize total travel time? Please note the journey begins at the first place listed.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "The order is Princeton University, Yale University, Harvard University"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Princeton University, Yale University, Harvard University"
+ },
+ "intent_template_id": 65
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 77,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
+ "instantiation_dict": {
+ "status": "Pending"
+ },
+ "intent": "What is the total count of Pending reviews amongst all the reviews?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "5"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "5"
+ },
+ "intent_template_id": 277
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 78,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
+ "instantiation_dict": {
+ "status": "Approved"
+ },
+ "intent": "What is the total count of Approved reviews amongst all the reviews?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "346"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "346"
+ },
+ "intent_template_id": 277
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 79,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What is the total count of {{status}} reviews amongst all the reviews?",
+ "instantiation_dict": {
+ "status": "Not Approved"
+ },
+ "intent": "What is the total count of Not Approved reviews amongst all the reviews?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 277
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 80,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
+ "instantiation_dict": {
+ "place_A": "Carnegie Mellon University",
+ "place_B": "Starbucks on Craig Street",
+ "place_C": "Pittsburgh International Airport"
+ },
+ "intent": "What is the duration required to first walk from Carnegie Mellon University to Starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "38 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "38 min"
+ },
+ "intent_template_id": 72
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 81,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
+ "instantiation_dict": {
+ "place_A": "Univ of Pittsburgh",
+ "place_B": "starbucks on Craig Street",
+ "place_C": "Pittsburgh International Airport"
+ },
+ "intent": "What is the duration required to first walk from Univ of Pittsburgh to starbucks on Craig Street, and then drive to Pittsburgh International Airport?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "49 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "49 min"
+ },
+ "intent_template_id": 72
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 82,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
+ "instantiation_dict": {
+ "place_A": "Massachusetts Institute of Technology",
+ "place_B": "Harvard University",
+ "place_C": "Boston Logan International Airport"
+ },
+ "intent": "What is the duration required to first walk from Massachusetts Institute of Technology to Harvard University, and then drive to Boston Logan International Airport?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "63 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "63 min"
+ },
+ "intent_template_id": 72
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 83,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the duration required to first walk from {{place_A}} to {{place_B}}, and then drive to {{place_C}}?",
+ "instantiation_dict": {
+ "place_A": "Carnegie Mellon University",
+ "place_B": "apple store shadyside",
+ "place_C": "starbucks on craig street"
+ },
+ "intent": "What is the duration required to first walk from Carnegie Mellon University to apple store shadyside, and then drive to starbucks on craig street?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "22 min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "22 min"
+ },
+ "intent_template_id": 72
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 84,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
+ "instantiation_dict": {
+ "hotel": "DoubleTree by Hilton New York Downtown",
+ "place": "Keens Steakhouse"
+ },
+ "intent": "From my stay at DoubleTree by Hilton New York Downtown, what's the estimated driving time to reach Keens Steakhouse?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "14 minutes"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "14 minutes"
+ },
+ "intent_template_id": 64
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 85,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
+ "instantiation_dict": {
+ "hotel": "La Quinta Inn near the airport",
+ "place": "Carnegie Mellon University"
+ },
+ "intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Carnegie Mellon University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "30 minutes"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "30 minutes"
+ },
+ "intent_template_id": 64
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 86,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
+ "instantiation_dict": {
+ "hotel": "La Quinta Inn near the airport",
+ "place": "Upitt"
+ },
+ "intent": "From my stay at La Quinta Inn near the airport, what's the estimated driving time to reach Upitt?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "29 minutes"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "29 minutes"
+ },
+ "intent_template_id": 64
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 87,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
+ "instantiation_dict": {
+ "hotel": "red roof inn",
+ "place": "Pittsburgh science museum"
+ },
+ "intent": "From my stay at red roof inn, what's the estimated driving time to reach Pittsburgh science museum?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "20 minutes"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "20 minutes"
+ },
+ "intent_template_id": 64
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 88,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "From my stay at {{hotel}}, what's the estimated driving time to reach {{place}}?",
+ "instantiation_dict": {
+ "hotel": "Homewood Suites Southpointe",
+ "place": "PPG Paints Arena"
+ },
+ "intent": "From my stay at Homewood Suites Southpointe, what's the estimated driving time to reach PPG Paints Arena?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "34 minutes"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "34 minutes"
+ },
+ "intent_template_id": 64
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 89,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Which US states border {{state}}?",
+ "instantiation_dict": {
+ "state": "Connecticut"
+ },
+ "intent": "Which US states border Connecticut?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Rhode Island",
+ "Massachusetts",
+ "New York"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Rhode Island, Massachusetts, New York"
+ },
+ "intent_template_id": 67
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 90,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Which US states border {{state}}?",
+ "instantiation_dict": {
+ "state": "Pennsylvania"
+ },
+ "intent": "Which US states border Pennsylvania?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Ohio",
+ "Maryland",
+ "New York",
+ "New Jersey",
+ "Delaware",
+ "West Virginia"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Ohio, Maryland, New York, New Jersey, Delaware, West Virginia"
+ },
+ "intent_template_id": 67
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 91,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Which US states border {{state}}?",
+ "instantiation_dict": {
+ "state": "Massachusetts"
+ },
+ "intent": "Which US states border Massachusetts?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Rhode Island",
+ "Connecticut",
+ "New York",
+ "New Hampshire",
+ "Vermont"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Rhode Island, Connecticut, New York, New Hampshire, Vermont"
+ },
+ "intent_template_id": 67
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 92,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Which US states border {{state}}?",
+ "instantiation_dict": {
+ "state": "Vermont"
+ },
+ "intent": "Which US states border Vermont?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "New York",
+ "New Hampshire",
+ "Massachusetts"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "New York, New Hampshire, Massachusetts"
+ },
+ "intent_template_id": 67
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 93,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Which US states border {{state}}?",
+ "instantiation_dict": {
+ "state": "New Hampshire"
+ },
+ "intent": "Which US states border New Hampshire?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Massachusetts",
+ "Vermont",
+ "Maine"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Massachusetts, Vermont, Maine"
+ },
+ "intent_template_id": 67
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 94,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Telll me the grand total of invoice {{id}}.",
+ "instantiation_dict": {
+ "id": "000000001"
+ },
+ "intent": "Telll me the grand total of invoice 000000001.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "36.39"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$36.39"
+ },
+ "intent_template_id": 274
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 95,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Telll me the grand total of invoice {{id}}.",
+ "instantiation_dict": {
+ "id": "000000002"
+ },
+ "intent": "Telll me the grand total of invoice 000000002.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "39.64"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$39.64"
+ },
+ "intent_template_id": 274
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 96,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the status of my latest order and when will it arrive",
+ "instantiation_dict": {},
+ "intent": "Tell me the status of my latest order and when will it arrive",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "The last order was canceled. It will never arrive."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "reference_answer_raw_annotation": "The last order was canceled. It will never arrive.",
+ "string_note": ""
+ },
+ "intent_template_id": 193
+ },
+ {
+ "sites": [
+ "map",
+ "wikipedia"
+ ],
+ "task_id": 97,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
+ "instantiation_dict": {},
+ "intent": "Tell me the distance to drive from Carnegie Mellon University to the top computer science school in massachusetts",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "914km |OR| 914 km"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "914 km"
+ },
+ "intent_template_id": 120
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 98,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
+ "instantiation_dict": {
+ "places": "tea cafe",
+ "start": "University of Pittsburgh"
+ },
+ "intent": "Where is the nearest tea cafe to University of Pittsburgh, and what is the walking distance to it?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Fuku Tea",
+ "3716",
+ "Forbes Avenue",
+ "Central Oakland",
+ "Pittsburgh",
+ "653m"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Fuku Tea, 3716, Forbes Avenue, Oakland, Central Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n653m"
+ },
+ "intent_template_id": 66
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 99,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
+ "instantiation_dict": {
+ "places": "Five Guys",
+ "start": "5700 Penn Ave"
+ },
+ "intent": "Where is the nearest Five Guys to 5700 Penn Ave, and what is the walking distance to it?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Five Guys",
+ "117",
+ "South Bouquet Street",
+ "North Oakland",
+ "Pittsburgh",
+ "4.0km"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Five Guys, 117, South Bouquet Street, Oakland, North Oakland, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n4.0km"
+ },
+ "intent_template_id": 66
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 100,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
+ "instantiation_dict": {
+ "places": "Starbucks",
+ "start": "Carnegie Mellon"
+ },
+ "intent": "Where is the nearest Starbucks to Carnegie Mellon, and what is the walking distance to it?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Starbucks",
+ "417",
+ "South Craig Street",
+ "Bellefield",
+ "Pittsburgh",
+ "557m"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Starbucks, 417, South Craig Street, Bellefield, Pittsburgh, Allegheny County, Pennsylvania, 15213, United States\n557m"
+ },
+ "intent_template_id": 66
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 101,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{places}} to {{start}}, and what is the walking distance to it?",
+ "instantiation_dict": {
+ "places": "In-N-Out",
+ "start": "Upitts"
+ },
+ "intent": "Where is the nearest In-N-Out to Upitts, and what is the walking distance to it?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no In-N-Out near University of Pittsburgh",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 66
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 102,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
+ "instantiation_dict": {
+ "label": "help needed",
+ "repo": "a11yproject/a11yproject.com"
+ },
+ "intent": "Display the list of issues in the a11yproject/a11yproject.com repository that have labels related to help needed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/?label_name%5B%5D=help%20wanted",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 349
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 103,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
+ "instantiation_dict": {
+ "label": "questions",
+ "repo": "kkroening/ffmpeg-python"
+ },
+ "intent": "Display the list of issues in the kkroening/ffmpeg-python repository that have labels related to questions",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/kkroening/ffmpeg-python/-/issues/?label_name%5B%5D=question",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 349
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 104,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
+ "instantiation_dict": {
+ "label": "flaky-test",
+ "repo": "keycloak/keycloak"
+ },
+ "intent": "Display the list of issues in the keycloak/keycloak repository that have labels related to flaky-test",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/keycloak/keycloak/-/issues/?label_name%5B%5D=flaky-test",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 349
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 105,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
+ "instantiation_dict": {
+ "label": "OpenAPI Generator CLI",
+ "repo": "OpenAPITools/openapi-generator"
+ },
+ "intent": "Display the list of issues in the OpenAPITools/openapi-generator repository that have labels related to OpenAPI Generator CLI",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/OpenAPITools/openapi-generator/-/issues/?label_name%5B%5D=OpenAPI%20Generator%20CLI",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 349
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 106,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Display the list of issues in the {{repo}} repository that have labels related to {{label}}",
+ "instantiation_dict": {
+ "label": "BUG",
+ "repo": "umano/AndroidSlidingUpPanel"
+ },
+ "intent": "Display the list of issues in the umano/AndroidSlidingUpPanel repository that have labels related to BUG",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/umano/AndroidSlidingUpPanel/-/issues/?label_name%5B%5D=BUG",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 349
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 107,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
+ "instantiation_dict": {
+ "period": "from May to December 2022"
+ },
+ "intent": "Presents the monthly count of successful orders from May to December 2022 in MM:COUNT format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "May: 8 orders",
+ "June: 13 orders",
+ "July: 9 orders",
+ "August: 8 orders",
+ "Sepetember: 10 orders",
+ "October: 4 orders",
+ "November: 5 orders",
+ "December: 10 orders"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
+ },
+ "intent_template_id": 270
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 108,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
+ "instantiation_dict": {
+ "period": "01/2023-05/2023"
+ },
+ "intent": "Presents the monthly count of successful orders 01/2023-05/2023 in MM:COUNT format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "January: 12 orders",
+ "Feburary: 7 orders",
+ "March: 5 orders",
+ "April: 9 orders",
+ "May: 5 orders"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "January: 12 orders Febulary: 7 orders March: 5 orders Apirl: 9 orders May: 5 orders"
+ },
+ "intent_template_id": 270
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 109,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
+ "instantiation_dict": {
+ "period": "from Jan to December 2022"
+ },
+ "intent": "Presents the monthly count of successful orders from Jan to December 2022 in MM:COUNT format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "January: 11 orders",
+ "Feburary: 16 orders",
+ "March: 14 orders",
+ "April: 7 orders",
+ "May: 8 orders",
+ "June: 13 orders",
+ "July: 9 orders",
+ "August: 8 orders",
+ "Sepetember: 10 orders",
+ "Octorbor: 4 orders",
+ "November: 5 orders",
+ "December: 10 orders"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders December: 10 orders "
+ },
+ "intent_template_id": 270
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 110,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
+ "instantiation_dict": {
+ "period": "from Jan to Nov 2022"
+ },
+ "intent": "Presents the monthly count of successful orders from Jan to Nov 2022 in MM:COUNT format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "January: 11 orders",
+ "Feburary: 16 orders",
+ "March: 14 orders",
+ "April: 7 orders",
+ "May: 8 orders",
+ "June: 13 orders",
+ "July: 9 orders",
+ "August: 8 orders",
+ "Sepetember: 10 orders",
+ "Octorbor: 4 orders",
+ "November: 5 orders"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "January: 11 orders Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
+ },
+ "intent_template_id": 270
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 111,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Presents the monthly count of successful orders {{period}} in MM:COUNT format",
+ "instantiation_dict": {
+ "period": "from Feb to Nov 2022"
+ },
+ "intent": "Presents the monthly count of successful orders from Feb to Nov 2022 in MM:COUNT format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Feburary: 16 orders",
+ "March: 14 orders",
+ "April: 7 orders",
+ "May: 8 orders",
+ "June: 13 orders",
+ "July: 9 orders",
+ "August: 8 orders",
+ "Sepetember: 10 orders",
+ "Octorbor: 4 orders",
+ "November: 5 orders"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Feburary: 16 orders March: 14 orders April: 7 orders May: 8 orders June: 13 orders July: 9 orders August: 8 orders Sepetember: 10 orders Octorbor: 4 orders November: 5 orders "
+ },
+ "intent_template_id": 270
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 112,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
+ "instantiation_dict": {
+ "product": "Circe fleece"
+ },
+ "intent": "Show me the customers who have expressed dissatisfaction with Circe fleece?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Hannah Lim"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Hannah Lim"
+ },
+ "intent_template_id": 245
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 113,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
+ "instantiation_dict": {
+ "product": "Olivia zip jacket"
+ },
+ "intent": "Show me the customers who have expressed dissatisfaction with Olivia zip jacket?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Emma Lopez",
+ "Seam Miller"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Emma Lopez, Seam Miller"
+ },
+ "intent_template_id": 245
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 114,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the customers who have expressed dissatisfaction with {{product}}?",
+ "instantiation_dict": {
+ "product": "Antonia racer tank"
+ },
+ "intent": "Show me the customers who have expressed dissatisfaction with Antonia racer tank?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Shaunte",
+ "Merrie"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Shaunte, Merrie"
+ },
+ "intent_template_id": 245
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 115,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}",
+ "instantiation_dict": {
+ "product": "Chloe tank"
+ },
+ "intent": "Show me the name of the customers who have expressed dissatisfaction with Chloe tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no negative review for Chloe tank",
+ "reference_answer_raw_annotation": ""
+ },
+ "intent_template_id": 245
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 116,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the name of the customers who have expressed dissatisfaction with {{product}}?",
+ "instantiation_dict": {
+ "product": "tanks products"
+ },
+ "intent": "Show me the name of the customers who have expressed dissatisfaction with tanks products?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Alexander",
+ "Carma",
+ "Dominic",
+ "Merrie",
+ "Monroe",
+ "Scotty",
+ "Shaunte",
+ "Teofila",
+ "Valorie"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Alexander, Carma, Dominic, Merrie, Monroe, Scotty, Shaunte, Teofila, Valorie"
+ },
+ "intent_template_id": 245
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 117,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the date when I made my first purchase on this site?",
+ "instantiation_dict": {},
+ "intent": "What is the date when I made my first purchase on this site?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "3/2/22"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3/2/22"
+ },
+ "intent_template_id": 161
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 118,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have jaw bruxism problem, show me something that could alleviate the problem.",
+ "instantiation_dict": {},
+ "intent": "I have jaw bruxism problem, show me something that could alleviate the problem.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "jaw bruxism",
+ "mouth guard"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 151
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 119,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the reasons why customers like {{product}}",
+ "instantiation_dict": {
+ "product": "Antonia Racer Tank"
+ },
+ "intent": "Tell me the reasons why customers like Antonia Racer Tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Its color and style is good"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Its color and style is good"
+ },
+ "intent_template_id": 250
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 120,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the reasons why customers like {{product}}",
+ "instantiation_dict": {
+ "product": "Ana Running Short"
+ },
+ "intent": "Tell me the reasons why customers like Ana Running Short",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "It is comfortable"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "It is comfortable"
+ },
+ "intent_template_id": 250
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 121,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the reasons why customers like {{product}}",
+ "instantiation_dict": {
+ "product": "Circe hooded fleece"
+ },
+ "intent": "Tell me the reasons why customers like Circe hooded fleece",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Warm and comfortable. True to size."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Warm and comfortable. True to size."
+ },
+ "intent_template_id": 250
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 122,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the reasons why customers like {{product}}",
+ "instantiation_dict": {
+ "product": "Olivia zip jacket"
+ },
+ "intent": "Tell me the reasons why customers like Olivia zip jacket",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Lightweight, comfortable and stylish. Good design and details."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lightweight, comfortable, and stylish. Good design and details."
+ },
+ "intent_template_id": 250
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 123,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the reasons why customers like {{product}}",
+ "instantiation_dict": {
+ "product": "Circe's products"
+ },
+ "intent": "Tell me the reasons why customers like Circe's products",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Warm and comfortable. True to size."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Warm and comfortable. True to size."
+ },
+ "intent_template_id": 250
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 124,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range of {{product}} in the One Stop Market?",
+ "instantiation_dict": {
+ "product": "wireless earphone"
+ },
+ "intent": "What is the price range of wireless earphone in the One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0.14",
+ "745.00"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$0.14 - $745.00"
+ },
+ "intent_template_id": 159
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 125,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range of {{product}} in the One Stop Market?",
+ "instantiation_dict": {
+ "product": "teeth grinding mouth guard"
+ },
+ "intent": "What is the price range of teeth grinding mouth guard in the One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1.46",
+ "85"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$1.46 - $85"
+ },
+ "intent_template_id": 159
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 126,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range of {{product}} in the One Stop Market?",
+ "instantiation_dict": {
+ "product": "Canon photo printer"
+ },
+ "intent": "What is the price range of Canon photo printer in the One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2.56",
+ "649.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$2.56 - $649.99"
+ },
+ "intent_template_id": 159
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 127,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What brands appear most frequently among the top search terms?",
+ "instantiation_dict": {},
+ "intent": "What brands appear most frequently among the top search terms?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Hollister",
+ "Joust",
+ "Antonia"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Hollister, Joust, Antonia"
+ },
+ "intent_template_id": 1001
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 128,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
+ "instantiation_dict": {
+ "k": "2"
+ },
+ "intent": "What's the total number of items sold in the most recent 2 orders?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "9"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "9"
+ },
+ "intent_template_id": 1002
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 129,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
+ "instantiation_dict": {
+ "k": "4"
+ },
+ "intent": "What's the total number of items sold in the most recent 4 orders?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "16"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "16"
+ },
+ "intent_template_id": 1002
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 130,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
+ "instantiation_dict": {
+ "k": "5"
+ },
+ "intent": "What's the total number of items sold in the most recent 5 orders?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "18"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "18"
+ },
+ "intent_template_id": 1002
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 131,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What's the total number of items sold in the most recent {{k}} orders?",
+ "instantiation_dict": {
+ "k": "7"
+ },
+ "intent": "What's the total number of items sold in the most recent 7 orders?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "25"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "25"
+ },
+ "intent_template_id": 1002
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 132,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
+ "instantiation_dict": {
+ "user": "kilian",
+ "repo": "a11yproject",
+ "date": "3/5/2023"
+ },
+ "intent": "How many commits did kilian make to a11yproject on 3/5/2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 322
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 133,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
+ "instantiation_dict": {
+ "user": "Eric",
+ "repo": "a11yproject",
+ "date": "3/2"
+ },
+ "intent": "How many commits did Eric make to a11yproject on 3/2?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2"
+ },
+ "intent_template_id": 322
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 134,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
+ "instantiation_dict": {
+ "user": "kilian",
+ "repo": "a11yproject",
+ "date": "3/1/2023"
+ },
+ "intent": "How many commits did kilian make to a11yproject on 3/1/2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 322
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 135,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
+ "instantiation_dict": {
+ "user": "Eric and Kilian",
+ "repo": "a11yproject",
+ "date": "1/3/2023"
+ },
+ "intent": "How many commits did Eric and Kilian make to a11yproject on 1/3/2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 322
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 136,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make to {{repo}} on {{date}}?",
+ "instantiation_dict": {
+ "user": "Steven Woodson",
+ "repo": "a11y-webring.club",
+ "date": "2/6/2023"
+ },
+ "intent": "How many commits did Steven Woodson make to a11y-webring.club on 2/6/2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "5"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "5"
+ },
+ "intent_template_id": 322
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 137,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
+ "instantiation_dict": {
+ "city1": "the city where the Liberty Bell is located",
+ "city2": "the home city of Pirates"
+ },
+ "intent": "What is the estimated driving time between the city where the Liberty Bell is located and the home city of Pirates?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "5h 47min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "5h 47min"
+ },
+ "intent_template_id": 51
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 138,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
+ "instantiation_dict": {
+ "city1": "the big apple",
+ "city2": "the city with the most authentic Philly cheesesteaks"
+ },
+ "intent": "What is the estimated driving time between the big apple and the city with the most authentic Philly cheesesteaks?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "1h 58min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1h 58min"
+ },
+ "intent_template_id": 51
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 139,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
+ "instantiation_dict": {
+ "city1": "the hometown of Joe Biden",
+ "city2": "Bridgeport"
+ },
+ "intent": "What is the estimated driving time between the hometown of Joe Biden and Bridgeport?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "3h 20min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3h 20min"
+ },
+ "intent_template_id": 51
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 140,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the estimated driving time between {{city1}} and {{city2}}?",
+ "instantiation_dict": {
+ "city1": "the city of Niagara Falls",
+ "city2": "the city of Yale University"
+ },
+ "intent": "What is the estimated driving time between the city of Niagara Falls and the city of Yale University?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "8h 33min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "8h 33min"
+ },
+ "intent_template_id": 51
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 141,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spent on {{category}} shopping during {{time}}",
+ "instantiation_dict": {
+ "category": "food-related",
+ "time": "March 2023"
+ },
+ "intent": "How much I spent on food-related shopping during March 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "47.41"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$47.41"
+ },
+ "intent_template_id": 162
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 142,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spent on {{category}} shopping during {{time}}",
+ "instantiation_dict": {
+ "category": "hair care and hair style",
+ "time": "Jan 2023"
+ },
+ "intent": "How much I spent on hair care and hair style shopping during Jan 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "95.23"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$95.23"
+ },
+ "intent_template_id": 162
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 143,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spent on {{category}} shopping during {{time}}",
+ "instantiation_dict": {
+ "category": "home decoration",
+ "time": "1/29/2023"
+ },
+ "intent": "How much I spent on home decoration shopping during 1/29/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "265.69"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$265.69"
+ },
+ "intent_template_id": 162
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 144,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spent on {{category}} shopping during {{time}}",
+ "instantiation_dict": {
+ "category": "food",
+ "time": "from mid Jan to the end Jan 2023"
+ },
+ "intent": "How much I spent on food shopping during from mid Jan to the end Jan 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 162
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 145,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spent on {{category}} shopping during {{time}}",
+ "instantiation_dict": {
+ "category": "cooking and food",
+ "time": "March 2022"
+ },
+ "intent": "How much I spent on cooking and food shopping during March 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "52.35"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$52.35"
+ },
+ "intent_template_id": 162
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 146,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
+ "instantiation_dict": {
+ "option": "size",
+ "product": "picture frame",
+ "time": "Sep 2022"
+ },
+ "intent": "What is the size configuration of the picture frame I bought Sep 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "16x24"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "16x24"
+ },
+ "intent_template_id": 155
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 147,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
+ "instantiation_dict": {
+ "option": "size",
+ "product": "picture frame",
+ "time": "2022"
+ },
+ "intent": "What is the size configuration of the picture frame I bought 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "16x24"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "16x24"
+ },
+ "intent_template_id": 155
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 148,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
+ "instantiation_dict": {
+ "option": "color",
+ "product": "picture frame",
+ "time": "Sep 2022"
+ },
+ "intent": "What is the color configuration of the picture frame I bought Sep 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Mist"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Mist"
+ },
+ "intent_template_id": 155
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 149,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
+ "instantiation_dict": {
+ "option": "color",
+ "product": "artifical plants",
+ "time": "Feb 2023"
+ },
+ "intent": "What is the color configuration of the artifical plants I bought Feb 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Green-vines"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Green-vines"
+ },
+ "intent_template_id": 155
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 150,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the {{option}} configuration of the {{product}} I bought {{time}}",
+ "instantiation_dict": {
+ "option": "price",
+ "product": "fake tree",
+ "time": "Jan 2023"
+ },
+ "intent": "What is the price configuration of the fake tree I bought Jan 2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "260.69"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "260.69"
+ },
+ "intent_template_id": 155
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 151,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
+ "instantiation_dict": {
+ "location1": "CMU",
+ "location2": "University of Pittsburgh"
+ },
+ "intent": "What is the minimum travel time by car from CMU to University of Pittsburgh?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "4min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "4min"
+ },
+ "intent_template_id": 36
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 152,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
+ "instantiation_dict": {
+ "location1": "Schenley park",
+ "location2": "Upitt"
+ },
+ "intent": "What is the minimum travel time by car from Schenley park to Upitt?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "4min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "4min"
+ },
+ "intent_template_id": 36
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 153,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
+ "instantiation_dict": {
+ "location1": "REI",
+ "location2": "CMU"
+ },
+ "intent": "What is the minimum travel time by car from REI to CMU?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "7min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "7min"
+ },
+ "intent_template_id": 36
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 154,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
+ "instantiation_dict": {
+ "location1": "CMU gates building",
+ "location2": "Schenley park"
+ },
+ "intent": "What is the minimum travel time by car from CMU gates building to Schenley park?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "4min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "4min"
+ },
+ "intent_template_id": 36
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 155,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the minimum travel time by car from {{location1}} to {{location2}}?",
+ "instantiation_dict": {
+ "location1": "Animal Rescue League of Pittsburgh",
+ "location2": "Schenley park"
+ },
+ "intent": "What is the minimum travel time by car from Animal Rescue League of Pittsburgh to Schenley park?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "9min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "9min"
+ },
+ "intent_template_id": 36
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 156,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Checkout merge requests assigned to me",
+ "instantiation_dict": {},
+ "intent": "Checkout merge requests assigned to me",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/dashboard/merge_requests?assignee_username=byteblaze",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 290
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 157,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show all customers",
+ "instantiation_dict": {},
+ "intent": "Show all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/customer/index/",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 255
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 158,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
+ "instantiation_dict": {
+ "num": 11
+ },
+ "intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 11 cards",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 171
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 159,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
+ "instantiation_dict": {
+ "num": 31
+ },
+ "intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 31 cards",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 171
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 160,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
+ "instantiation_dict": {
+ "num": 6
+ },
+ "intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 6 cards",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 171
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 161,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
+ "instantiation_dict": {
+ "num": 23
+ },
+ "intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 23 cards",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/heiying-game-card-case-for-nintendo-switch-switch-oled-game-card-or-micro-sd-memory-cards-portable-switch-game-memory-card-storage-with-24-game-card-slots-and-24-micro-sd-card-slots-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 171
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 162,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all {{num}} cards",
+ "instantiation_dict": {
+ "num": 40
+ },
+ "intent": "I have a lot of Nintendo Switch game cards now, help me find the best storage option to fit all 40 cards",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 171
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 163,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/ostent-16gb-memory-card-stick-storage-for-sony-ps-vita-psv1000-2000-pch-z081-z161-z321-z641.html",
+ "geolocation": null,
+ "intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "instantiation_dict": {},
+ "intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "I ordered the 16gb but I only got 14 gigs even though I formatted the card",
+ "The memory card is kind of slow on games and downloads",
+ "No original packaging It's used and the previous owners data has not been erased",
+ "The product is a legit sony hardware that have been owned by someone else before",
+ "The media could not be loaded",
+ "I could not format the card so I wasn\u2019t able to use it for my VITA"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "I ordered the 16gb but I only got 14 gigs even though I formatted the card. The memory card is kind of slow on games and downloads. No original packaging It's used and the previous owners data has not been erased. The product is a legit sony hardware that have been owned by someone else before The media could not be loaded. I could not format the card so I wasn\u2019t able to use it for my VITA"
+ },
+ "intent_template_id": 136
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 164,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/mineralogie-all-natural-lip-gloss-ruby-rose.html",
+ "geolocation": null,
+ "intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "instantiation_dict": {},
+ "intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Dry",
+ "Uneven color"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "DryUneven color"
+ },
+ "intent_template_id": 136
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 165,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/sandgrens-swedish-handmade-wooden-clog-sandal-copenhagen.html",
+ "geolocation": null,
+ "intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "instantiation_dict": {},
+ "intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "The 39 was too small. I am afraid the 40 will be too big",
+ "I was very sad when the shoe rubbed up against my baby toe",
+ "I had to return them because I knew in time it would tear up my feet",
+ "The problem is that the strap is made of some really stiff leather and is painful to my heel",
+ "The front is also uncomfortably tight",
+ "The Dansko's were similar (not as bad) and loosened up over time"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "The 39 was too small. I am afraid the 40 will be too big. I was very sad when the shoe rubbed up against my baby toe. I had to return them because I knew in time it would tear up my feet. The problem is that the strap is made of some really stiff leather and is painful to my heel. The front is also uncomfortably tight. The Dansko's were similar (not as bad) and loosened up over time."
+ },
+ "intent_template_id": 136
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 166,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/sensodyne-repair-protect-whitening-toothpaste-with-fluoride-3-4-oz-pack-of-3.html",
+ "geolocation": null,
+ "intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "instantiation_dict": {},
+ "intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "there is no existing criticism",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 136
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 167,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/photosmart-plus-b209-clr-inkjetfb-p-s-c-usb-wrls-1.html",
+ "geolocation": null,
+ "intent_template": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "instantiation_dict": {},
+ "intent": "What are the main criticisms of this product? Please extract the relevant sentences.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "The wireless connection works on a whim (about 40% of the time I've owned it)",
+ "It seems to constantly run out of ink",
+ "Cartridge prices are less than some printers I've had",
+ "This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason",
+ "Scanner is as slow as my first scanner I ever owned in the mid-90's",
+ "For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "The wireless connection works on a whim (about 40% of the time I've owned it). It seems to constantly run out of ink. Cartridge prices are less than some printers I've had, but now I understand why. This printer seems to have more reasons NOT to work (none that are findable or correctable) Ex: error boxes saying that it's out of paper when it automatically switches to photo printing for some reason. Scanner is as slow as my first scanner I ever owned in the mid-90's. For the $176 I paid, there isn't even a fax component on it. I guess the \"PLUS\" part of it's name is in reference to the migraines it causes when you can't figure out the new reason why it's not working for the 10th time in the past 2 months."
+ },
+ "intent_template_id": 136
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 168,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
+ "instantiation_dict": {
+ "description": "more than 100"
+ },
+ "intent": "Tell me the full names of the repositories where I made contributions and they got more than 100 stars?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "No repo found",
+ "reference_answer_raw_annotation": "No repo found"
+ },
+ "intent_template_id": 289
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 169,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
+ "instantiation_dict": {
+ "description": "the most"
+ },
+ "intent": "Tell me the full names of the repositories where I made contributions and they got the most stars?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "a11yproject.com",
+ "design"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "a11yproject.com, Primer/design"
+ },
+ "intent_template_id": 289
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 170,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
+ "instantiation_dict": {
+ "description": "the least"
+ },
+ "intent": "Tell me the full names of the repositories where I made contributions and they got the least stars?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "cloud-to-butt",
+ "dotfiles",
+ "timeit",
+ "solarized-prism-theme",
+ "gimmiethat.space",
+ "remove-board-movement-events-from-the-github-issue-timeline"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
+ },
+ "intent_template_id": 289
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 171,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
+ "instantiation_dict": {
+ "description": "less than 5"
+ },
+ "intent": "Tell me the full names of the repositories where I made contributions and they got less than 5 stars?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "a11y-syntax-highlighting",
+ "a11y-webring.club",
+ "accessible-html-content-patterns",
+ "ericwbailey.website",
+ "cloud-to-butt",
+ "dotfiles",
+ "timeit",
+ "solarized-prism-theme",
+ "gimmiethat.space",
+ "remove-board-movement-events-from-the-github-issue-timeline"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "a11y-syntax-highlighting, a11y-webring.club, accessible-html-content-patterns, ericwbailey.website, cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
+ },
+ "intent_template_id": 289
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 172,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me the full names of the repositories where I made contributions and they got {{description}} stars?",
+ "instantiation_dict": {
+ "description": "no"
+ },
+ "intent": "Tell me the full names of the repositories where I made contributions and they got no stars?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "cloud-to-butt",
+ "dotfiles",
+ "timeit",
+ "solarized-prism-theme",
+ "gimmiethat.space",
+ "remove-board-movement-events-from-the-github-issue-timeline"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline"
+ },
+ "intent_template_id": 289
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 173,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "better"
+ },
+ "intent": "Open my latest updated issue that has keyword \"better\" in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/8",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": "",
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 310
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 174,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "feature"
+ },
+ "intent": "Open my latest updated issue that has keyword \"feature\" in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/a11y-webring.club/-/issues/71",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": ""
+ },
+ "intent_template_id": 310
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 175,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "dependency"
+ },
+ "intent": "Open my latest updated issue that has keyword \"dependency\" in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": ""
+ },
+ "intent_template_id": 310
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 176,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "theme editor"
+ },
+ "intent": "Open my latest updated issue that has keyword \"theme editor\" in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": ""
+ },
+ "intent_template_id": 310
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 177,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest updated issue that has keyword \"{{keyword}}\" in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "homepage content"
+ },
+ "intent": "Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["Yes, it is closed"]
+ },
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
+ "program_html": [],
+ "reference_answer_raw_annotation": "closed",
+ "string_note": ""
+ },
+ "intent_template_id": 310
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 178,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "better"
+ },
+ "intent": "Open my latest created issue that has better in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["Yes, it is closed"]
+ },
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/566",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Closed",
+ "string_note": ""
+ },
+ "intent_template_id": 500
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 179,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "feature"
+ },
+ "intent": "Open my latest created issue that has feature in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["Yes, it is closed"]
+ },
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1517",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Closed",
+ "string_note": ""
+ },
+ "intent_template_id": 500
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 180,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "dependency"
+ },
+ "intent": "Open my latest created issue that has dependency in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues/18",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": ""
+ },
+ "intent_template_id": 500
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 181,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "theme editor"
+ },
+ "intent": "Open my latest created issue that has theme editor in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["No, it is open"]
+ },
+ "reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues/1",
+ "program_html": [],
+ "reference_answer_raw_annotation": "Not closed",
+ "string_note": ""
+ },
+ "intent_template_id": 500
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 182,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open my latest created issue that has {{keyword}} in its title to check if it is closed",
+ "instantiation_dict": {
+ "keyword": "homepage content"
+ },
+ "intent": "Open my latest created issue that has homepage content in its title to check if it is closed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match",
+ "url_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["Yes, it is closed"]
+ },
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/719",
+ "program_html": [],
+ "reference_answer_raw_annotation": "closed",
+ "string_note": ""
+ },
+ "intent_template_id": 500
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 183,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
+ "instantiation_dict": {
+ "Attribute": "SKU",
+ "N": "10"
+ },
+ "intent": "Give me the SKU of the products that have 10 units left",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no product that has 10 quantities left.",
+ "reference_answer_raw_annotation": "There is no product that has 10 quantities left."
+ },
+ "intent_template_id": 368
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 184,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
+ "instantiation_dict": {
+ "Attribute": "name",
+ "N": "0"
+ },
+ "intent": "Give me the name of the products that have 0 units left",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Sinbad Fitness Tank"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Sinbad Fitness Tank"
+ },
+ "intent_template_id": 368
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 185,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
+ "instantiation_dict": {
+ "Attribute": "brand",
+ "N": "3"
+ },
+ "intent": "Give me the brand of the products that have 3 units left",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Eos",
+ "Minerva"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Eos, Minerva"
+ },
+ "intent_template_id": 368
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 186,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
+ "instantiation_dict": {
+ "Attribute": "product names and the sizes",
+ "N": "2-3"
+ },
+ "intent": "Give me the product names and the sizes of the products that have 2-3 units left",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Eos V-Neck Hoodie: S",
+ "Minera Luma Tech V-Tee: XS"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Eos V-Neck Hoodie: S Minera Luma Tech V-Tee: XS"
+ },
+ "intent_template_id": 368
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 187,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Give me the {{Attribute}} of the products that have {{N}} units left",
+ "instantiation_dict": {
+ "Attribute": "SKU",
+ "N": "1-3"
+ },
+ "intent": "Give me the SKU of the products that have 1-3 units left",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "WH11-S-Blue",
+ "WS08-XS-Blue"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "WH11-S-Blue, WS08-XS-Blue"
+ },
+ "intent_template_id": 368
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 188,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the total cost of my latest {{status}} order?",
+ "instantiation_dict": {
+ "status": "cancelled"
+ },
+ "intent": "Tell me the total cost of my latest cancelled order?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "365.42"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "365.42"
+ },
+ "intent_template_id": 214
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 189,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the total cost of my latest {{status}} order?",
+ "instantiation_dict": {
+ "status": "pending"
+ },
+ "intent": "Tell me the total cost of my latest pending order?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "754.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "754.99"
+ },
+ "intent_template_id": 214
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 190,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the total cost of my latest {{status}} order?",
+ "instantiation_dict": {
+ "status": "complete"
+ },
+ "intent": "Tell me the total cost of my latest complete order?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "65.32"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "65.32"
+ },
+ "intent_template_id": 214
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 191,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the total cost of my latest {{status}} order?",
+ "instantiation_dict": {
+ "status": "processing"
+ },
+ "intent": "Tell me the total cost of my latest processing order?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no order of \"processing\" status",
+ "reference_answer_raw_annotation": "There is no order of \"processing\" status"
+ },
+ "intent_template_id": 214
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 192,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me the total cost of my latest {{status}} order?",
+ "instantiation_dict": {
+ "status": "non-cancelled"
+ },
+ "intent": "Tell me the total cost of my latest non-cancelled order?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "754.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "754.99"
+ },
+ "intent_template_id": 214
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 193,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
+ "instantiation_dict": {
+ "status": "completed",
+ "N": "2"
+ },
+ "intent": "Get the total payment amount of the last 2 completed orders",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "182.4"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "182.4"
+ },
+ "intent_template_id": 367
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 194,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
+ "instantiation_dict": {
+ "status": "completed",
+ "N": "5"
+ },
+ "intent": "Get the total payment amount of the last 5 completed orders",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "555.2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "555.2"
+ },
+ "intent_template_id": 367
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 195,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
+ "instantiation_dict": {
+ "status": "pending",
+ "N": "5"
+ },
+ "intent": "Get the total payment amount of the last 5 pending orders",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "885.4"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "885.4"
+ },
+ "intent_template_id": 367
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 196,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Compare the payment difference of the last {{N}} {{status_1}} orders and {{status_2}} orders",
+ "instantiation_dict": {
+ "status_1": "cancelled",
+ "status_2": "completed",
+ "N": "4"
+ },
+ "intent": "Compare the payment difference of the last 4 cancelled orders and completed orders",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "194.25"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "194.25"
+ },
+ "intent_template_id": 367
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 197,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the total payment amount of the last {{N}} {{status}} orders",
+ "instantiation_dict": {
+ "status": "non-cancelled",
+ "N": "5"
+ },
+ "intent": "Get the total payment amount of the last 5 non-cancelled orders",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "778.2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "annotation_note": "219.4+210+166.4+93.4+89",
+ "reference_answer_raw_annotation": "778.2"
+ },
+ "intent_template_id": 367
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 198,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "customer name",
+ "status": "most recent cancelled"
+ },
+ "intent": "Get the customer name of the most recent cancelled order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Lily Potter"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lily Potter"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 199,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "order ID",
+ "status": "newest pending"
+ },
+ "intent": "Get the order ID of the newest pending order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "299"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "299"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 200,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "billing name",
+ "status": "oldest complete"
+ },
+ "intent": "Get the billing name of the oldest complete order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "John Lee"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "John Lee"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 201,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "customer name",
+ "status": "earliest fraud suspect"
+ },
+ "intent": "Get the customer name of the earliest fraud suspect order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no order of \"fraud suspect\" status",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 202,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "date",
+ "status": "most recent canlled"
+ },
+ "intent": "Get the date of the most recent canlled order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "May 23 2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "May 23, 2023"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 203,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "purchase date and order id",
+ "status": "most recent pending"
+ },
+ "intent": "Get the purchase date and order id of the most recent pending order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "order id: 000000299",
+ "purchase date: May 31, 2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "000000299, May 31, 2023, 2:55:09 AM"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 204,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Get the {{attribute}} of the {{status}} order",
+ "instantiation_dict": {
+ "attribute": "product name and discounted price (low to high)",
+ "status": "most recent completed"
+ },
+ "intent": "Get the product name and discounted price (low to high) of the most recent completed order",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Rapha Sports Short: $35",
+ "Thorpe Track Pant: $54.4",
+ "Mach Street Sweatshirt: $62"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Rapha Sports Short: $35 Thorpe Track Pant: $54.4 Mach Street Sweatshirt: $62"
+ },
+ "intent_template_id": 366
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 205,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make on {{date}}?",
+ "instantiation_dict": {
+ "user": "kilian",
+ "date": "3/5/2023"
+ },
+ "intent": "How many commits did kilian make on 3/5/2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 320
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 206,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make on {{date}}?",
+ "instantiation_dict": {
+ "user": "Eric",
+ "date": "3/2"
+ },
+ "intent": "How many commits did Eric make on 3/2?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "2"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2"
+ },
+ "intent_template_id": 320
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 207,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make on {{date}} in total?",
+ "instantiation_dict": {
+ "user": "Eric and Kilian",
+ "date": "1/3/2023"
+ },
+ "intent": "How many commits did Eric and Kilian make on 1/3/2023 in total?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 320
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 208,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
+ "instantiation_dict": {
+ "PhoneNum": "+1 2058812302"
+ },
+ "intent": "Find the customer name and email with phone number +1 2058812302",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "John Smith",
+ "john.smith.xyz@gmail.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "John Smith, john.smith.xyz@gmail.com"
+ },
+ "intent_template_id": 364
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 209,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
+ "instantiation_dict": {
+ "PhoneNum": "2137418080"
+ },
+ "intent": "Find the customer name and email with phone number 2137418080",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Jennifer White",
+ "jennifer.white@yahoo.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Jennifer White, jennifer.white@yahoo.com"
+ },
+ "intent_template_id": 364
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 210,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
+ "instantiation_dict": {
+ "PhoneNum": "2065555555"
+ },
+ "intent": "Find the customer name and email with phone number 2065555555",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Adam Garcia",
+ "gamingpro456@gmail.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Adam Garcia, gamingpro456@gmail.com"
+ },
+ "intent_template_id": 364
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 211,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
+ "instantiation_dict": {
+ "PhoneNum": "8015551212"
+ },
+ "intent": "Find the customer name and email with phone number 8015551212",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Sean Miller",
+ "sean.miller@gmail.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Sean Miller, sean.miller@gmail.com"
+ },
+ "intent_template_id": 364
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 212,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Find the customer name and email with phone number {{PhoneNum}}",
+ "instantiation_dict": {
+ "PhoneNum": "555-229-3326"
+ },
+ "intent": "Find the customer name and email with phone number 555-229-3326",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Veronica Costello",
+ "roni_cost@example.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Veronica Costello, roni_cost@example.com"
+ },
+ "intent_template_id": 364
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 213,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the key aspects that the customers don't like about {{product}}",
+ "instantiation_dict": {
+ "product": "Antonia Racer Tank"
+ },
+ "intent": "What are the key aspects that the customers don't like about Antonia Racer Tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Not suitable for high-impact workouts"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Not suitable for high-impact workouts"
+ },
+ "intent_template_id": 249
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 214,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the key aspects that the customers don't like about {{product}}",
+ "instantiation_dict": {
+ "product": "Zing Jump Rope"
+ },
+ "intent": "What are the key aspects that the customers don't like about Zing Jump Rope",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "It is hard to find the right size. Won't last long"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "It is hard to find the right size. Won't last long"
+ },
+ "intent_template_id": 249
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 215,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the key aspects that the customers don't like about {{product}}",
+ "instantiation_dict": {
+ "product": "Circe ice fleece"
+ },
+ "intent": "What are the key aspects that the customers don't like about Circe ice fleece",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Material quality, fit, insufficient warmth, color"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Material quality, fit, insufficient warmth, color"
+ },
+ "intent_template_id": 249
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 216,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the key aspects that the customers don't like about {{product}}",
+ "instantiation_dict": {
+ "product": "Electra Bra Top"
+ },
+ "intent": "What are the key aspects that the customers don't like about Electra Bra Top",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Not true to size"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Not true to size"
+ },
+ "intent_template_id": 249
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 217,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "What are the key aspects that the customers don't like about {{product}}",
+ "instantiation_dict": {
+ "product": "Pursuit Tone Band"
+ },
+ "intent": "What are the key aspects that the customers don't like about Pursuit Tone Band",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Insufficient resistance for their workouts."
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Insufficient resistance for their workouts."
+ },
+ "intent_template_id": 249
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 218,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
+ "instantiation_dict": {
+ "location": "CMU, Pittsburgh",
+ "n": "5"
+ },
+ "intent": "Show me the walking distance from nearby hotels to CMU, Pittsburgh that take at most 5 minutes?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 41
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 219,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
+ "instantiation_dict": {
+ "location": "Pittsburgh airport",
+ "n": "3"
+ },
+ "intent": "Show me the walking distance from nearby hotels to Pittsburgh airport that take at most 3 minutes?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no hotel near CMU that is within 5 minutes walking distance",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 41
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 220,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the walking distance from nearby hotels to {{location}} that take at most {{n}} minutes?",
+ "instantiation_dict": {
+ "location": "Gardner Steel Conference Center,",
+ "n": 5
+ },
+ "intent": "Show me the walking distance from nearby hotels to Gardner Steel Conference Center, that take at most 5 minutes?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Wyndham Pittsburgh University Cente: 375m",
+ "The Oaklander Hotel: 338m"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Wyndham Pittsburgh University Cente: 375 m\nThe Oaklander Hotel: 338 m"
+ },
+ "intent_template_id": 41
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 221,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am at CMU Pittsburgh, how long it takes to the nearest {{location}} with different transportation methods?",
+ "instantiation_dict": {
+ "location": "USPS postal office"
+ },
+ "intent": "I am at CMU Pittsburgh, how long it takes to the nearest USPS postal office with different transportation methods?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Walk: 1 minute",
+ "Drive: less than 1 minute",
+ "Bike: less than 1 minute"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Walk: 1 minute to walk and\nDrive: less than 1 minute\nBike: less than 1 minute"
+ },
+ "intent_template_id": 35
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 222,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
+ "instantiation_dict": {
+ "location": "cold stone ice cream"
+ },
+ "intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest cold stone ice cream",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "3min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3min"
+ },
+ "intent_template_id": 35
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 223,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
+ "instantiation_dict": {
+ "location": "Mcdonald's"
+ },
+ "intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest Mcdonald's",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "4min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "4min"
+ },
+ "intent_template_id": 35
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 224,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am at CMU Pittsburgh, how long it takes to drive to the nearest {{location}}",
+ "instantiation_dict": {
+ "location": "wendys"
+ },
+ "intent": "I am at CMU Pittsburgh, how long it takes to drive to the nearest wendys",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "3min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3min"
+ },
+ "intent_template_id": 35
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 225,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What do customers say about {{product_type}} from {{manufature}}",
+ "instantiation_dict": {
+ "product_type": "brush",
+ "manufature": "sephora"
+ },
+ "intent": "What do customers say about brush from sephora",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The sephora brushes don't have reviews",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 135
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 226,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range for products from {{brand}}?",
+ "instantiation_dict": {
+ "brand": "Amazon basic"
+ },
+ "intent": "What is the price range for products from Amazon basic?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "5.49",
+ "375.19"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$5.49 - $375.19"
+ },
+ "intent_template_id": 370
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 227,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range for products from {{brand}}?",
+ "instantiation_dict": {
+ "brand": "EYZUTAK"
+ },
+ "intent": "What is the price range for products from EYZUTAK?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "9.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$9.99"
+ },
+ "intent_template_id": 370
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 228,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range for products from {{brand}}?",
+ "instantiation_dict": {
+ "brand": "sephora"
+ },
+ "intent": "What is the price range for products from sephora?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "18.18",
+ "94.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$18.18 - $94.99"
+ },
+ "intent_template_id": 370
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 229,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range for products from {{brand}}?",
+ "instantiation_dict": {
+ "brand": "ugreen"
+ },
+ "intent": "What is the price range for products from ugreen?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "6.99",
+ "38.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$6.99 - $38.99"
+ },
+ "intent_template_id": 370
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 230,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the price range for products from {{brand}}?",
+ "instantiation_dict": {
+ "brand": "Perricone MD"
+ },
+ "intent": "What is the price range for products from Perricone MD?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "35",
+ "149"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$35 - $149"
+ },
+ "intent_template_id": 370
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 231,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Get the order number of my most recent {{status}} order ",
+ "instantiation_dict": {
+ "status": "cancelled"
+ },
+ "intent": "Get the order number of my most recent cancelled order ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "170"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "000000170"
+ },
+ "intent_template_id": 213
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 232,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Get the order number of my most recent {{status}} order ",
+ "instantiation_dict": {
+ "status": "pending"
+ },
+ "intent": "Get the order number of my most recent pending order ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "189"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "000000189"
+ },
+ "intent_template_id": 213
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 233,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Get the order number of my most recent {{status}} order ",
+ "instantiation_dict": {
+ "status": "complete"
+ },
+ "intent": "Get the order number of my most recent complete order ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "180"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "000000180"
+ },
+ "intent_template_id": 213
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 234,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Get the order number of my most recent {{status}} order ",
+ "instantiation_dict": {
+ "status": "on hold"
+ },
+ "intent": "Get the order number of my most recent on hold order ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "there is no on hold order",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 213
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 235,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Get the order number of my most recent {{status}} order ",
+ "instantiation_dict": {
+ "status": "under delivery"
+ },
+ "intent": "Get the order number of my most recent under delivery order ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no under delivery order",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 213
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 236,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
+ "instantiation_dict": {
+ "location": "pharmacy",
+ "location2": "Carnegie Mellon",
+ "condition": "I can walk within 20mins"
+ },
+ "intent": "Where is the nearest pharmacy from Carnegie Mellon I can walk within 20mins",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Schiller's Pharmacy",
+ "811",
+ "South Aiken Avenue",
+ "Shadyside",
+ "Pittsburgh"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Schiller's Pharmacy, 811, South Aiken Avenue, Shadyside, Pittsburgh, Allegheny County, 15232, United States"
+ },
+ "intent_template_id": 39
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 237,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Where is the nearest {{location}} from {{location2}} {{condition}}",
+ "instantiation_dict": {
+ "location": "gas station",
+ "location2": "CMU",
+ "condition": ""
+ },
+ "intent": "Where is the nearest gas station from CMU ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Sunoco",
+ "North Craig Street",
+ "North Oakland",
+ "Pittsburgh"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Sunoco, North Craig Street, North Oakland, Pittsburgh, Allegheny County, 15213, United States"
+ },
+ "intent_template_id": 39
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 238,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
+ "instantiation_dict": {
+ "product_category": "PS4 accessories"
+ },
+ "intent": "I am doing a market survey for one stop market, show me the most expensive product from PS4 accessories category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/astro-gaming-a50-wireless-headset-base-station-gen-4-compatible-with-ps5-ps4-pc-mac-black-silver.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 138
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 239,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
+ "instantiation_dict": {
+ "product_category": "nutrition bars and drinks"
+ },
+ "intent": "I am doing a market survey for one stop market, show me the most expensive product from nutrition bars and drinks category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/kellogg-s-special-k-protein-meal-bars-chocolate-caramel-12-7oz-6-count.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 138
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 240,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
+ "instantiation_dict": {
+ "product_category": "competitive swimwear"
+ },
+ "intent": "I am doing a market survey for one stop market, show me the most expensive product from competitive swimwear category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/women-cross-flower-beachwear-tankini-bandeau-bandage-bikini-set-push-up-swimwear-bathing-suit-two-pieces-swimsuits.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 138
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 241,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
+ "instantiation_dict": {
+ "product_category": "skin care tool"
+ },
+ "intent": "I am doing a market survey for one stop market, show me the most expensive product from skin care tool category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/professional-medi-spa-scar-stretch-mark-reduction-system.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 138
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 242,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
+ "instantiation_dict": {
+ "product_category": "Household Supplies"
+ },
+ "intent": "I am doing a market survey for one stop market, show me the most expensive product from Household Supplies category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/lynx-battery-12v-200ah-lithium-iron-phosphate-lifepo4-prismatic-deep-cell-battery-set-of-4-3-2v-cells-with-3-bus-bars-and-8-lug-nuts-for-rv-solar-marine-off-grid-applications.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 138
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 243,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
+ "instantiation_dict": {
+ "information": "email address",
+ "product": "Circe fleece"
+ },
+ "intent": "Show me the email address of the customer who is the most unhappy with Circe fleece",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "hannah.lim@gmail.com"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "hannah.lim@gmail.com"
+ },
+ "intent_template_id": 244
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 244,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
+ "instantiation_dict": {
+ "information": "email address",
+ "product": "Olivia zip jacket"
+ },
+ "intent": "Show me the email address of the customer who is the most unhappy with Olivia zip jacket",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "emma.lopez@gmail.com"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "emma.lopez@gmail.com"
+ },
+ "intent_template_id": 244
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 245,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
+ "instantiation_dict": {
+ "information": "name",
+ "product": "Antonia racer tank"
+ },
+ "intent": "Show me the name of the customer who is the most unhappy with Antonia racer tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Shaunte"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Shaunte"
+ },
+ "intent_template_id": 244
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 246,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
+ "instantiation_dict": {
+ "information": "name",
+ "product": "Chloe tank"
+ },
+ "intent": "Show me the name of the customer who is the most unhappy with Chloe tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Teofila"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Teofila"
+ },
+ "intent_template_id": 244
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 247,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Show me the {{information}} of the customer who is the most unhappy with {{product}}",
+ "instantiation_dict": {
+ "information": "email address",
+ "product": "the style of Zoe products"
+ },
+ "intent": "Show me the email address of the customer who is the most unhappy with the style of Zoe products",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "Valorie doesn't have a email in the system",
+ "program_html": [],
+ "string_note": "There is no negative review for Zoe products",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 244
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 248,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the coordinates of {{location}} in DD format",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon Caf\u00e9"
+ },
+ "intent": "Tell me the coordinates of Carnegie Mellon Caf\u00e9 in DD format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.442",
+ "-79.939"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.4424191, -79.9397388"
+ },
+ "intent_template_id": 46
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 249,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the coordinates of {{location}} in DD format",
+ "instantiation_dict": {
+ "location": "Western Pennsylvania Hospital Heliport"
+ },
+ "intent": "Tell me the coordinates of Western Pennsylvania Hospital Heliport in DD format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.460",
+ "-79.946"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.46076, -79.94666"
+ },
+ "intent_template_id": 46
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 250,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the coordinates of {{location}} in DD format",
+ "instantiation_dict": {
+ "location": "Apple Store near Pitt"
+ },
+ "intent": "Tell me the coordinates of Apple Store near Pitt in DD format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.451",
+ "-79.933"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.4511693, -79.9334241"
+ },
+ "intent_template_id": 46
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 251,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the coordinates of {{location}} in DD format",
+ "instantiation_dict": {
+ "location": "bus stop on the Carnegie art museum side of the street near CMU"
+ },
+ "intent": "Tell me the coordinates of bus stop on the Carnegie art museum side of the street near CMU in DD format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.444",
+ "-79.948"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.4443, -79.94889"
+ },
+ "intent_template_id": 46
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 252,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Tell me the coordinates of {{location}} in DD format",
+ "instantiation_dict": {
+ "location": "Tokyo Japanese Food Store in Pittsburgh"
+ },
+ "intent": "Tell me the coordinates of Tokyo Japanese Food Store in Pittsburgh in DD format",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.457",
+ "-79.929"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.45761, -79.92934"
+ },
+ "intent_template_id": 46
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 253,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the {{information}} of {{location}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon Caf\u00e9",
+ "information": "phone number"
+ },
+ "intent": "What is the phone number of Carnegie Mellon Caf\u00e9",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no such information in the map",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 501
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 254,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the {{information}} of {{location}}",
+ "instantiation_dict": {
+ "location": "Western Pennsylvania Hospital",
+ "information": "phone number"
+ },
+ "intent": "What is the phone number of Western Pennsylvania Hospital",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ["4125785000"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "4125785000"
+ },
+ "intent_template_id": 501
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 255,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Who is the {{information}} of {{location}}",
+ "instantiation_dict": {
+ "location": "PIT airport",
+ "information": "operator"
+ },
+ "intent": "Who is the operator of PIT airport",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Allegheny County Airport Authority"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Allegheny County Airport Authority"
+ },
+ "intent_template_id": 501
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 256,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the {{information}} of {{location}}",
+ "instantiation_dict": {
+ "location": "Carnegie art museum in pittsburgh",
+ "information": "website"
+ },
+ "intent": "What is the website of Carnegie art museum in pittsburgh",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "http://web.cmoa.org/"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "http://web.cmoa.org/"
+ },
+ "intent_template_id": 501
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 257,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What is the {{information}} of {{location}}",
+ "instantiation_dict": {
+ "location": "Tokyo Japanese Food Store in Pittsburgh",
+ "information": "hours of operation"
+ },
+ "intent": "What is the hours of operation of Tokyo Japanese Food Store in Pittsburgh",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "We-Su 10:00-17:00"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "We-Su 10:00-17:00"
+ },
+ "intent_template_id": 501
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 258,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "See all public projects",
+ "instantiation_dict": {},
+ "intent": "See all public projects",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/explore",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 325
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 259,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Get me my RSS feed token",
+ "instantiation_dict": {},
+ "intent": "Get me my RSS feed token",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "TMN_bBn9Z48qVbUFZV45"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "TMN_bBn9Z48qVbUFZV45"
+ },
+ "intent_template_id": 312
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 260,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I want to browse the products in the {{category}} category",
+ "instantiation_dict": {
+ "category": "Video Game"
+ },
+ "intent": "I want to browse the products in the Video Game category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/video-games.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 211
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 261,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I want to browse the products in the {{category}} category",
+ "instantiation_dict": {
+ "category": "Headphones"
+ },
+ "intent": "I want to browse the products in the Headphones category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/electronics/headphones.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/electronics.html?cat=60"}]
+ },
+ "intent_template_id": 211
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 262,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I want to browse the products in the {{category}} category",
+ "instantiation_dict": {
+ "category": "Men shoes"
+ },
+ "intent": "I want to browse the products in the Men shoes category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=145"}]
+ },
+ "intent_template_id": 211
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 263,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I want to browse the products in the {{category}} category",
+ "instantiation_dict": {
+ "category": "Woman clothing"
+ },
+ "intent": "I want to browse the products in the Woman clothing category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/clothing.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=143"}]
+ },
+ "intent_template_id": 211
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 264,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I want to browse the products in the {{category}} category",
+ "instantiation_dict": {
+ "category": "Cabinets, Racks & Shelves"
+ },
+ "intent": "I want to browse the products in the Cabinets, Racks & Shelves category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/office-products/office-furniture-lighting/cabinets-racks-shelves.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/office-products.html?cat=187"}]
+ },
+ "intent_template_id": 211
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 265,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What's the closest national park to {{city}}? How far is it to drive there?",
+ "instantiation_dict": {
+ "city": "Boston"
+ },
+ "intent": "What's the closest national park to Boston? How far is it to drive there?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Acadia National Park",
+ "457km"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Acadia National Park\n457km"
+ },
+ "intent_template_id": 85
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 266,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What's the closest national park to {{city}}?",
+ "instantiation_dict": {
+ "city": "the largest city in Maine"
+ },
+ "intent": "What's the closest national park to the largest city in Maine?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Acadia National Park"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Acadia National Park"
+ },
+ "intent_template_id": 85
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 267,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What's the closest national park to {{city}}? How long it takes to drive there?",
+ "instantiation_dict": {
+ "city": "the hometown of Stephen King"
+ },
+ "intent": "What's the closest national park to the hometown of Stephen King? How long it takes to drive there?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Acadia National Park"
+ ],
+ "fuzzy_match": [
+ "1h 23min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Acadia National Park\n1h 23min"
+ },
+ "intent_template_id": 85
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 268,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "What's the closest national park to {{city}}? How long does it take to bike there?",
+ "instantiation_dict": {
+ "city": "Vinalhaven, ME"
+ },
+ "intent": "What's the closest national park to Vinalhaven, ME? How long does it take to bike there?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Acadia National Park"
+ ],
+ "fuzzy_match": [
+ "10h 33min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Acadia National Park\n10h 33min"
+ },
+ "intent_template_id": 85
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 269,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
+ "instantiation_dict": {
+ "price": "25",
+ "product_category": "women shoes"
+ },
+ "intent": "Show me products under $25 in \"women shoes\" category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/clothing-shoes-jewelry/women/shoes.html?price=0-25",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 139
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 270,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
+ "instantiation_dict": {
+ "price": "30",
+ "product_category": "men shoes"
+ },
+ "intent": "Show me products under $30 in \"men shoes\" category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/clothing-shoes-jewelry/men/shoes.html?price=0-30",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 139
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 271,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
+ "instantiation_dict": {
+ "price": "46.99",
+ "product_category": "makeup remover"
+ },
+ "intent": "Show me products under $46.99 in \"makeup remover\" category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/beauty-personal-care/makeup/makeup-remover.html?price=0-46.99",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 139
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 272,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
+ "instantiation_dict": {
+ "price": "78",
+ "product_category": "children dental care"
+ },
+ "intent": "Show me products under $78 in \"children dental care\" category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/beauty-personal-care/oral-care/children-s-dental-care.html?price=0-78",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 139
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 273,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me products under ${{price}} in \"{{product_category}}\" category",
+ "instantiation_dict": {
+ "price": "199",
+ "product_category": "furtiture with accent"
+ },
+ "intent": "Show me products under $199 in \"furtiture with accent\" category",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/home-kitchen/furniture/accent-furniture.html?price=0-199",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 139
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 274,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Search for \"{{keyword}}\"",
+ "instantiation_dict": {
+ "keyword": "usb wifi"
+ },
+ "intent": "Search for \"usb wifi\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=usb+wifi",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 212
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 275,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Search for \"{{keyword}}\"",
+ "instantiation_dict": {
+ "keyword": "xbox"
+ },
+ "intent": "Search for \"xbox\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=xbox",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 212
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 276,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Search for \"{{keyword}}\"",
+ "instantiation_dict": {
+ "keyword": "switch accessories"
+ },
+ "intent": "Search for \"switch accessories\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=switch+accessories",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 212
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 277,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Search for \"{{keyword}}\"",
+ "instantiation_dict": {
+ "keyword": "batteries for iphone 13"
+ },
+ "intent": "Search for \"batteries for iphone 13\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=iphone+13",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 212
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 278,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Search for \"{{keyword}}\"",
+ "instantiation_dict": {
+ "keyword": "green tea bag for weight loss"
+ },
+ "intent": "Search for \"green tea bag for weight loss\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=green+tea+bag+for+weight+loss",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 212
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 279,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
+ "instantiation_dict": {},
+ "intent": "Provide me with the complete names of Bluetooth headphones from Sony, and also share the price range for the available models",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed)",
+ "Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones",
+ "Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus",
+ "Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black",
+ "Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items)",
+ "Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W)",
+ "Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B)",
+ "Sony MDRAS600BT Active Sports Bluetooth Headset (Black)",
+ "Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items)",
+ "Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R)",
+ "Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black",
+ "SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed",
+ "18.99",
+ "406"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "These models are avaiable: SONY WH1000XM3 Bluetooth Wireless Noise Canceling Headphones Silver WH-1000XM3/S (Renewed) Sony WH-CH710N/H Wireless Bluetooth Noise Cancelling Headphones Sony WH-1000XM3B Wireless Bluetooth Noise-Canceling Over-Ear Headphones (Black) Basic Headphone Bundle Kit with Stylus Sony Wireless Headphones WH-CH510: Wireless Bluetooth On-Ear Headset with Mic for Phone-Call, Black Sony WHCH710N Wireless Bluetooth Noise Canceling Over-The-Ear Headphones (Black) with Kratos 18W PD Two-Port Power Adapter and Kratos 6-Feet Nylon Braided USB-C Cable Bundle (3 Items) Sony WI-SP500 Wireless in-Ear Sports Headphones, White (WISP500/W) Sony WI-SP510 Extra BASS Wireless in-Ear Headset/Headphones with mic for Phone Call Sports IPX5 Bluetooth, Black (WISP510/B) Sony MDRAS600BT Active Sports Bluetooth Headset (Black) Sony WH-1000XM4 Wireless Noise Canceling Over-Ear Headphones (Black) with Sony WLA-NS7 Wireless TV Adapter Bundle (2 Items) Sony WI-C300 Wireless In-Ear Headphones, Red (WIC300/R) Sony XB950N1 Extra Bass Wireless Noise Canceling Headphones, Black SONY - H900N Hi-Res Noise Cancelling Wireless Headphone Grayish Black Renewed The price ranges from $18.99 to $406 "
+ },
+ "intent_template_id": 204
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 280,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
+ "instantiation_dict": {},
+ "intent": "Provide me with the full names of chargers from Anker, and also share the price range for the available models",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More",
+ "Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included)",
+ "Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278)",
+ "5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply",
+ "Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter)",
+ "Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter)",
+ "USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More",
+ "iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy",
+ "USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More",
+ "Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111)",
+ "Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More",
+ "USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included)",
+ "8.99",
+ "59.99"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "These models are availiable: Anker USB C Charger 30W, 711 Charger, Compact Fast Charger (Not Foldable) for MacBook Air/iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy S21, Note 20, iPad Pro, Pixel, and More Anker USB C Charger 40W, 521 Charger (Nano Pro), PIQ 3.0 Durable Compact Fast Charger (Not Foldable) for iPhone 13/13 Mini/13 Pro/13 Pro Max/12, Galaxy, Pixel 4/3, iPad/iPad Mini (Cable Not Included) Anker PowerCore Speed 20000, 20000mAh Qualcomm Quick Charge 3.0 & PowerIQ Portable Charger, with Quick Charge Recharging, Power Bank for Samsung, iPhone, iPad and More, Black (A1278) 5Ft Micro-USB Charger Cord Cable Fit for Anker-PowerCore 5000 10000 20100 13000 26800 Mini 3350 Fusion II 15000 Redux 20000 Slim 10000 Astro E1 AC Replacement Power Adapter Supply Anker 10W Max Wireless Charger, 313 Wireless Charger (Pad), Qi-Certified Wireless Charging 7.5W for iPhone 12/12 Pro/12 mini/12 Pro Max, 10W for Galaxy S10 S9 S8, S9 Plus, Note 9 (No AC Adapter) Anker Wireless Charger, 313 Wireless Charger (Stand), Qi-Certified for iPhone 12, 12 Pro Max, SE, 11, 11 Pro, 11 Pro Max, XR, XS Max, 10W Fast-Charging Galaxy S20, S10 (No AC Adapter) USB Charger, Anker Elite Dual Port 24W Wall Charger, PowerPort 2 with PowerIQ and Foldable Plug, for iPhone 11/Xs/XS Max/XR/X/8/7/6/Plus, iPad Pro/Air 2/Mini 3/Mini 4, Samsung S4/S5, and More iPhone 12 Charger [GaN Tech], Anker 30W Compact USB-C Wall Charger with Power Delivery, PowerPort Atom for iPhone 12 / Mini/Pro/Pro Max / 11 / X/XS/XR, iPad Pro, MacBook 12'', Pixel, Galaxy USB C Charger, Anker 30W 2 Port Fast Charger with 18W USB C Power Adapter, Foldable PowerPort PD 2 Charger for iPad Pro, iPhone 11/11 Pro / 11 Pro Max/XS/Max/XR/X, Pixel, Galaxy, and More Anker 40W 5-Port USB Wall Charger, PowerPort 5 for iPhone XS / XS Max / XR / X / 8 / 7 / 6 / Plus, iPad Pro / Air 2 / mini, Galaxy S9 / S8 / Edge / Plus, Note 8 / 7, LG, Nexus, HTC and More, Black (AK-A2124111) Anker Quick Charge 3.0 39W Dual USB Wall Charger, PowerPort Speed 2 for Galaxy S10/S9/S8/Edge/Plus, Note 8/7 and PowerIQ for iPhone Xs/XS Max/XR/X/8/Plus, iPad Pro/Air 2/Mini, LG, Nexus, HTC and More USB C Charger, Anker 20W PIQ 3.0 Fast Charger with Foldable Plug, PowerPort III Charger for iPhone 13/13 Mini/13 Pro/13 Pro Max/12/11, iPad/iPad Mini, MagSafe, and More (Cable Not Included) Magnetic Wireless Charger, Anker Wireless Charger with 5ft Built-in USB-C Cable, PowerWave Magnetic Pad, 7.5W Charging for iPhone 13 / 13 Pro / 13 Pro Max / 13 mini / 12 / 12 Pro (No AC Adapter) USB C Super Fast Charger, Anker 25W PD Wall Charger Fast Charging for Samsung Galaxy S21/S21+/S21 Ultra/S20/Z Flip/Note20/20 Ultra/Note10/10+/S9/S8/S10e, iPad Pro 12.9, and More (Cable not Included) The price ranges from $8.99 to $59.99"
+ },
+ "intent_template_id": 204
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 281,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
+ "instantiation_dict": {},
+ "intent": "Please provide me with the complete product names of Oral B brush heads designed for children, along with their corresponding price range per brush",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count",
+ "Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack",
+ "3.745",
+ "6.495"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "These models are availiable: Oral-B Kids Extra Soft Replacement Brush Heads featuring STAR WARS, 2 count Kids By Oral-b Stages Power Star Wars Replacement Heads 4 Pack The price ranges from $3.745 to $6.495 "
+ },
+ "intent_template_id": 204
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 282,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
+ "instantiation_dict": {},
+ "intent": "List the full product names of slide slippers from Nike and tell me the price range of the available products",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Nike Men's Air Max Camden Slide Sandal",
+ "Nike Men's Benassi JDI Fanny Pack Slides",
+ "Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10)",
+ "Nike Offcourt Slide Mens Bq4639-002 Size 12",
+ "Nike Jordan Men's Break Slide Red AR6374-602",
+ "Nike Victori One Slide Mens Style : Dd9559-300",
+ "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14)",
+ "Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8)",
+ "Nike womens Benassi Just Do It",
+ "27.6",
+ "90.65"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "These models are availiable: Nike Men's Air Max Camden Slide Sandal Nike Men's Benassi JDI Fanny Pack Slides Nike Victori One Mens Comfort Slide Cn9675-003 (Midnight Navy/Midnight Navy/White, Numeric_10) Nike Offcourt Slide Mens Bq4639-002 Size 12 Nike Jordan Men's Break Slide Red AR6374-602 Nike Victori One Slide Mens Style : Dd9559-300 Nike Men's Benassi Solarsoft Slide Athletic Sandal (Black/White, numeric_14) Nike Men's Benassi Solarsoft Slide Athletic Sandal (Midnight Navy/Blue, numeric_8) Nike womens Benassi Just Do It The price ranges from $27.6 to $90.65"
+ },
+ "intent_template_id": 204
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 283,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Look up the most recent models of XBox controllers released between 2020-2021?",
+ "instantiation_dict": {},
+ "intent": "Look up the most recent models of XBox controllers released between 2020-2021?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/microsoft-xbox-controller-carbon-black-for-series-x-series-s-xbox-one-windows-10-android-ios-bundled-with-dual-port-charging-dock-xbox-controller-skin-voucher-premgear-cloth.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 210
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 284,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
+ "instantiation_dict": {
+ "product": "shoe storage",
+ "min_storage": "12 pairs"
+ },
+ "intent": "Show the least expensive shoe storage with a minimum storage capacity of 12 pairs.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/onlyeasy-over-the-door-shoe-storage-organizer-hanging-shoe-rack-holder-with-24-large-fabric-pockets-22-1-x-61-4-herringbone-grey-mxrodsb1p.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 207
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 285,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
+ "instantiation_dict": {
+ "product": "switch card holder",
+ "min_storage": "15 cards"
+ },
+ "intent": "Show the least expensive switch card holder with a minimum storage capacity of 15 cards.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/game-card-holder-storage-case-for-nintendo-switch-games-or-ps-vita-game-case-or-sd-memory-cards-black.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 207
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 286,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the least expensive {{product}} with a minimum storage capacity of {{min_storage}}.",
+ "instantiation_dict": {
+ "product": "ssd hard drive",
+ "min_storage": "1TB"
+ },
+ "intent": "Show the least expensive ssd hard drive with a minimum storage capacity of 1TB.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/external-hard-drive-2tb-ultra-thin-external-hard-drive-2000gb-ultra-high-speed-portable-3-1-type-c-storage-drive-compatible-with-pc-laptop-and-mac-2tb-a1.html",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 207
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 287,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "How much time does it take from Pittsburgh to Philadelphia by car?",
+ "instantiation_dict": {},
+ "intent": "How much time does it take from Pittsburgh to Philadelphia by car?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "5h 47min"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "5h 47min"
+ },
+ "intent_template_id": 47
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 288,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
+ "instantiation_dict": {
+ "attribute": "name"
+ },
+ "intent": "Tell me the name of the customer who has the most cancellations in the history",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "Samantha Jones"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Samantha Jones"
+ },
+ "intent_template_id": 234
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 289,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
+ "instantiation_dict": {
+ "attribute": "email address, name, phone number"
+ },
+ "intent": "Tell me the email address, name, phone number of the customer who has the most cancellations in the history",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "email: coolcat321@hotmail.com",
+ "name: Samantha Jones",
+ "phone number: 3055551212"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "email: coolcat321@hotmail.com name: Samantha Jones phone number: 3055551212"
+ },
+ "intent_template_id": 234
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 290,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
+ "instantiation_dict": {
+ "attribute": "product SKUs in the most recent cancelled orders"
+ },
+ "intent": "Tell me the product SKUs in the most recent cancelled orders of the customer who has the most cancellations in the history",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "WSH09-29-White",
+ "WSH09-28-Green",
+ "MSH11-34-Blue",
+ "WP09-29-Purple"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "WSH09-29-White,WSH09-28-Green,MSH11-34-Blue,WP09-29-Purple"
+ },
+ "intent_template_id": 234
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 291,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
+ "instantiation_dict": {
+ "attribute": "total spend on products in the most recent cancelled orders"
+ },
+ "intent": "Tell me the total spend on products in the most recent cancelled orders of the customer who has the most cancellations in the history",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "148"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "$148"
+ },
+ "intent_template_id": 234
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 292,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the customer who has the most cancellations in the history",
+ "instantiation_dict": {
+ "attribute": "total number of cancellations"
+ },
+ "intent": "Tell me the total number of cancellations of the customer who has the most cancellations in the history",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "9"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "9"
+ },
+ "intent_template_id": 234
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 293,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Show me the command to clone {{repo}} with SSH.",
+ "instantiation_dict": {
+ "repo": "Super_Awesome_Robot"
+ },
+ "intent": "Show me the command to clone Super_Awesome_Robot with SSH.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/super_awesome_robot.git"
+ },
+ "intent_template_id": 329
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 294,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Show me the command to clone {{repo}} with SSH.",
+ "instantiation_dict": {
+ "repo": "ChatGPT"
+ },
+ "intent": "Show me the command to clone ChatGPT with SSH.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/convexegg/chatgpt.git"
+ },
+ "intent_template_id": 329
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 295,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Show me the command to clone {{repo}} with SSH.",
+ "instantiation_dict": {
+ "repo": "metaseq"
+ },
+ "intent": "Show me the command to clone metaseq with SSH.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "git clone ssh://git@metis.lti.cs.cmu.edu:2222/root/metaseq.git"
+ },
+ "intent_template_id": 329
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 296,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Show me the command to clone {{repo}} with SSH.",
+ "instantiation_dict": {
+ "repo": "the best GAN python implementation"
+ },
+ "intent": "Show me the command to clone the best GAN python implementation with SSH.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/eriklindernoren/PyTorch-GAN.git"
+ },
+ "intent_template_id": 329
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 297,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Show me the command to clone {{repo}} with SSH.",
+ "instantiation_dict": {
+ "repo": "the most stared Covid location tracker"
+ },
+ "intent": "Show me the command to clone the most stared Covid location tracker with SSH.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["git clone ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "ssh://git@metis.lti.cs.cmu.edu:2222/yjlou/2019-nCov.git"
+ },
+ "intent_template_id": 329
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 298,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the most recent {{status}} order page",
+ "instantiation_dict": {
+ "status": "completed"
+ },
+ "intent": "Show the most recent completed order page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/sales/order/view/order_id/180/",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 180
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 299,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the most recent {{status}} order page",
+ "instantiation_dict": {
+ "status": "cancelled"
+ },
+ "intent": "Show the most recent cancelled order page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/sales/order/view/order_id/170/",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 180
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 300,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the most recent {{status}} order page",
+ "instantiation_dict": {
+ "status": "pending"
+ },
+ "intent": "Show the most recent pending order page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/sales/order/view/order_id/189/",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 180
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 301,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the most recent {{status}} order page",
+ "instantiation_dict": {
+ "status": "processing"
+ },
+ "intent": "Show the most recent processing order page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {"fuzzy_match": "N/A"},
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "there is no order in processing"
+ },
+ "intent_template_id": 180
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 302,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show the most recent {{status}} order page",
+ "instantiation_dict": {
+ "status": "out of delivery"
+ },
+ "intent": "Show the most recent out of delivery order page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {"fuzzy_match": "N/A"},
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "there is no order in processing"
+ },
+ "intent_template_id": 180
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 303,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make {{period}}?",
+ "instantiation_dict": {
+ "user": "Kilian",
+ "period": "durning 2023"
+ },
+ "intent": "How many commits did Kilian make durning 2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "1"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1"
+ },
+ "intent_template_id": 321
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 304,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make {{period}}?",
+ "instantiation_dict": {
+ "user": "Eric",
+ "period": "between Feb 2023 and May 2023"
+ },
+ "intent": "How many commits did Eric make between Feb 2023 and May 2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "14"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "14"
+ },
+ "intent_template_id": 321
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 305,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make {{period}}?",
+ "instantiation_dict": {
+ "user": "Philip",
+ "period": "in 2023/1"
+ },
+ "intent": "How many commits did Philip make in 2023/1?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 321
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 306,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make {{period}}?",
+ "instantiation_dict": {
+ "user": "Anthony",
+ "period": "between 08/2022-09/2022"
+ },
+ "intent": "How many commits did Anthony make between 08/2022-09/2022?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 321
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 307,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "How many commits did {{user}} make {{period}}?",
+ "instantiation_dict": {
+ "user": "Nic",
+ "period": "in April 2021"
+ },
+ "intent": "How many commits did Nic make in April 2021?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "16"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "16"
+ },
+ "intent_template_id": 321
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 308,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
+ "instantiation_dict": {
+ "repo": "primer/design"
+ },
+ "intent": "Tell me who has made the most contributions, in terms of number of commits, to the primer/design project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Shawn Allen"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Shawn Allen"
+ },
+ "intent_template_id": 323
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 309,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
+ "instantiation_dict": {
+ "repo": "thoughtbot/administrate"
+ },
+ "intent": "Tell me who has made the most contributions, in terms of number of commits, to the thoughtbot/administrate project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Grayson Wright"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Grayson Wright"
+ },
+ "intent_template_id": 323
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 310,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
+ "instantiation_dict": {
+ "repo": "AndroidSlidingUpPanel"
+ },
+ "intent": "Tell me who has made the most contributions, in terms of number of commits, to the AndroidSlidingUpPanel project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["tokudu"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "tokudu"
+ },
+ "intent_template_id": 323
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 311,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
+ "instantiation_dict": {
+ "repo": "Pytorch GAN"
+ },
+ "intent": "Tell me who has made the most contributions, in terms of number of commits, to the Pytorch GAN project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Erik Linder-Nor\u00e9n"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Erik Linder-Nor\u00e9n"
+ },
+ "intent_template_id": 323
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 312,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Tell me who has made the most contributions, in terms of number of commits, to the {{repo}} project",
+ "instantiation_dict": {
+ "repo": "csvkit"
+ },
+ "intent": "Tell me who has made the most contributions, in terms of number of commits, to the csvkit project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["Christopher Groskopf"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Christopher Groskopf"
+ },
+ "intent_template_id": 323
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 313,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Which number to call for the customer service?",
+ "instantiation_dict": {},
+ "intent": "Which number to call for the customer service?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no phone number in the website",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 134
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 314,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
+ "instantiation_dict": {
+ "repo": "prime/design",
+ "attribute": "name"
+ },
+ "intent": "List the name of the top 3 contributors to prime/design repo, ranked by the number of commits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Shawn Allen",
+ "Inayaili Le\u00f3n",
+ "Aurora Pleguezuelo"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Shawn Allen, Inayaili Le\u00f3n, Aurora Pleguezuelo"
+ },
+ "intent_template_id": 324
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 315,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
+ "instantiation_dict": {
+ "repo": "Pytorch GAN",
+ "attribute": "email address"
+ },
+ "intent": "List the email address of the top 3 contributors to Pytorch GAN repo, ranked by the number of commits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "eriklindernoren@live.se",
+ "eriklindernoren@gmail.com",
+ "pinnacle.chen@qq.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "eriklindernoren@live.se, eriklindernoren@gmail.com, pinnacle.chen@qq.com"
+ },
+ "intent_template_id": 324
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 316,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
+ "instantiation_dict": {
+ "repo": "facebook's guide on building react apps",
+ "attribute": "name"
+ },
+ "intent": "List the name of the top 3 contributors to facebook's guide on building react apps repo, ranked by the number of commits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Ian Sutherland",
+ "Joe Hadda",
+ "Dan Abramov"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Ian Sutherland, Joe Hadda, Dan Abramov"
+ },
+ "intent_template_id": 324
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 317,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
+ "instantiation_dict": {
+ "repo": "metaseq",
+ "attribute": "name and number of commits"
+ },
+ "intent": "List the name and number of commits of the top 3 contributors to metaseq repo, ranked by the number of commits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Susan Zhang: 70",
+ "Stephen Roller: 51",
+ "Peter Albert: 12"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Susan Zhang: 70, Stephen Roller: 51, Peter Albert: 12"
+ },
+ "intent_template_id": 324
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 318,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "List the {{attribute}} of the top 3 contributors to {{repo}} repo, ranked by the number of commits?",
+ "instantiation_dict": {
+ "repo": "2019-nCov",
+ "attribute": "last names"
+ },
+ "intent": "List the last names of the top 3 contributors to 2019-nCov repo, ranked by the number of commits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Lo",
+ "Chen",
+ "Chu"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lo, Chen, Chu"
+ },
+ "intent_template_id": 324
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 319,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
+ "instantiation_dict": {
+ "time": "April 2022"
+ },
+ "intent": "How much refund I should expect from my order canlled in April 2022, including shipping fee",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 160
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 320,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
+ "instantiation_dict": {
+ "time": "Feb 2023"
+ },
+ "intent": "How much refund I should expect from my order canlled in Feb 2023, including shipping fee",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "406.53"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "406.53"
+ },
+ "intent_template_id": 160
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 321,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much refund I should expect from my order canlled in {{time}}, including shipping fee",
+ "instantiation_dict": {
+ "time": "2022"
+ },
+ "intent": "How much refund I should expect from my order canlled in 2022, including shipping fee",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "3053.97"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "3053.97"
+ },
+ "intent_template_id": 160
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 322,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much refund I should expect from my order canlled in {{time}} if I cannot get the shipping fee refunded?",
+ "instantiation_dict": {
+ "time": "May 2023"
+ },
+ "intent": "How much refund I should expect from my order canlled in May 2023 if I cannot get the shipping fee refunded?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "350.42"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "350.42"
+ },
+ "intent_template_id": 160
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 323,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much refund I should expect from my order canlled in {{time}}? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
+ "instantiation_dict": {
+ "time": "2022/03"
+ },
+ "intent": "How much refund I should expect from my order canlled in 2022/03? I only kept the AC-DC Adapter and the shop told me that I cannot get the shipping fee back",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "264.49"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "264.49"
+ },
+ "intent_template_id": 160
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 324,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
+ "instantiation_dict": {
+ "product": "chairs",
+ "sorting_order": "ascending price"
+ },
+ "intent": "Show me the \"chairs\" listings by ascending price.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=chairs&product_list_dir=asc",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 208
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 325,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
+ "instantiation_dict": {
+ "product": "mouth night guard",
+ "sorting_order": "descending price"
+ },
+ "intent": "Show me the \"mouth night guard\" listings by descending price.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/index/?q=mouth%20night%20guard%20&product_list_order=price",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 208
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 326,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
+ "instantiation_dict": {
+ "product": "Canon photo printer",
+ "sorting_order": "search relevance, from most to least"
+ },
+ "intent": "Show me the \"Canon photo printer\" listings by search relevance, from most to least.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/?q=Canon+photo+printer",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 208
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 327,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
+ "instantiation_dict": {
+ "product": "iphone 12 phone case",
+ "sorting_order": "name alphabetically"
+ },
+ "intent": "Show me the \"iphone 12 phone case\" listings by name alphabetically.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/index/?q=%20iphone%2012%20phone%20case&product_list_order=name",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 208
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 328,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the \"{{product}}\" listings by {{sorting_order}}.",
+ "instantiation_dict": {
+ "product": "iphone 12 phone case",
+ "sorting_order": "price"
+ },
+ "intent": "Show me the \"iphone 12 phone case\" listings by price.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/catalogsearch/result/index/?product_list_order=price&q=%20iphone%2012%20phone%20case",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 208
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 329,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
+ "instantiation_dict": {
+ "time": "on 4/19/2023"
+ },
+ "intent": "How much I spend on 4/19/2023 on shopping at One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 147
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 330,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
+ "instantiation_dict": {
+ "time": "in March 2023"
+ },
+ "intent": "How much I spend in March 2023 on shopping at One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "83.31"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "81.31"
+ },
+ "intent_template_id": 147
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 331,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
+ "instantiation_dict": {
+ "time": "in July 2022"
+ },
+ "intent": "How much I spend in July 2022 on shopping at One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "40.16"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "40.16"
+ },
+ "intent_template_id": 147
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 332,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much I spend {{time}} on shopping at One Stop Market?",
+ "instantiation_dict": {
+ "time": "each month from Jan to the end of March 2023"
+ },
+ "intent": "How much I spend each month from Jan to the end of March 2023 on shopping at One Stop Market?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "Jan: 572.8",
+ "Feb: 762.18",
+ "Mar: 83.31"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Jan: 572.8\nFeb: 762.18\nMar: 83.31"
+ },
+ "intent_template_id": 147
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 333,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "How much did I spend on shopping at One Stop Market {{time}}? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
+ "instantiation_dict": {
+ "time": "on November 2022"
+ },
+ "intent": "How much did I spend on shopping at One Stop Market on November 2022? They gave me a 20% discount on the total amount for orders exceeding $200 in cash",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "359.546"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "359.546"
+ },
+ "intent_template_id": 147
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 334,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me when I last ordered my {{description}}?",
+ "instantiation_dict": {
+ "description": "muffin cornbread mix"
+ },
+ "intent": "Tell me when I last ordered my muffin cornbread mix?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "March 11th 2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "March 11th 2023"
+ },
+ "intent_template_id": 169
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 335,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me when I last ordered my {{description}}?",
+ "instantiation_dict": {
+ "description": "body butter"
+ },
+ "intent": "Tell me when I last ordered my body butter?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "January 16th 2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "January 16th 2023"
+ },
+ "intent_template_id": 169
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 336,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me when I last ordered my {{description}}?",
+ "instantiation_dict": {
+ "description": "conditioner"
+ },
+ "intent": "Tell me when I last ordered my conditioner?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "January 16th 2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "January 16th 2023"
+ },
+ "intent_template_id": 169
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 337,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me when I last ordered my {{description}}?",
+ "instantiation_dict": {
+ "description": "bread olive"
+ },
+ "intent": "Tell me when I last ordered my bread olive?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "December 12th 2022"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "December 12th 2022"
+ },
+ "intent_template_id": 169
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 338,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Tell me when I last ordered my {{description}}?",
+ "instantiation_dict": {
+ "description": "toothpaste"
+ },
+ "intent": "Tell me when I last ordered my toothpaste?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "December 4th 2022"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "December 4th 2022"
+ },
+ "intent_template_id": 169
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 339,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "List all opened issues {{description}}",
+ "instantiation_dict": {
+ "description": "that report bugs"
+ },
+ "intent": "List all opened issues that report bugs",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/?label_name%5B%5D=bug",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 299
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 340,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "List all opened issues {{description}}",
+ "instantiation_dict": {
+ "description": "that report bugs"
+ },
+ "intent": "List all opened issues that report bugs",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/issues/?label_name%5B%5D=type%3A%20bug%20%F0%9F%90%9E",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 299
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 341,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/root/metaseq",
+ "geolocation": null,
+ "intent_template": "List all opened issues {{description}}",
+ "instantiation_dict": {
+ "description": "requesting new features"
+ },
+ "intent": "List all opened issues requesting new features",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=enhancement",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 299
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 342,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/root/metaseq",
+ "geolocation": null,
+ "intent_template": "List all opened issues {{description}}",
+ "instantiation_dict": {
+ "description": "that ask about OPT model related questions"
+ },
+ "intent": "List all opened issues that ask about OPT model related questions",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/root/metaseq/-/issues/?search=OPT&label_name%5B%5D=question",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 299
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 343,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/root/metaseq",
+ "geolocation": null,
+ "intent_template": "List all opened issues {{description}}",
+ "instantiation_dict": {
+ "description": "that don't have any labels"
+ },
+ "intent": "List all opened issues that don't have any labels",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/root/metaseq/-/issues/?label_name%5B%5D=None",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 299
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 344,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "How many reviews our shop received {{time}}?",
+ "instantiation_dict": {
+ "time": "by far"
+ },
+ "intent": "How many reviews our shop received by far?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "351"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "351"
+ },
+ "intent_template_id": 248
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 345,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "How many reviews our shop received {{time}}?",
+ "instantiation_dict": {
+ "time": "in Apr 2023"
+ },
+ "intent": "How many reviews our shop received in Apr 2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "351"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "351"
+ },
+ "intent_template_id": 248
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 346,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "How many reviews our shop received {{time}}?",
+ "instantiation_dict": {
+ "time": "during 2022"
+ },
+ "intent": "How many reviews our shop received during 2022?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 248
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 347,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "How many reviews our shop received {{time}}?",
+ "instantiation_dict": {
+ "time": "from the beginning of the shop"
+ },
+ "intent": "How many reviews our shop received from the beginning of the shop?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "351"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "351"
+ },
+ "intent_template_id": 248
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 348,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "How many reviews our shop received {{time}}?",
+ "instantiation_dict": {
+ "time": "in May 2023"
+ },
+ "intent": "How many reviews our shop received in May 2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 248
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 349,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
+ "instantiation_dict": {
+ "repo": "gimmiethat.space"
+ },
+ "intent": "Who else have access to my repo gimmiethat.space, show me their usernames",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "exact_match": "yjlou"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "yjlou"
+ },
+ "intent_template_id": 298
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 350,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Who else have access to my repo {{repo}}, show me their usernames",
+ "instantiation_dict": {
+ "repo": "prism-theme"
+ },
+ "intent": "Who else have access to my repo prism-theme, show me their usernames",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "abisubramanya27"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Abishek S, abisubramanya27"
+ },
+ "intent_template_id": 298
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 351,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List products from {{product_category}} category by {{order}} price",
+ "instantiation_dict": {
+ "product_category": "PS4 accessories",
+ "order": "ascending"
+ },
+ "intent": "List products from PS4 accessories category by ascending price",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/video-games/playstation-4/accessories.html?product_list_order=price",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/video-games.html?cat=67&product_list_order=price"}]
+ },
+ "intent_template_id": 137
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 352,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List products from {{product_category}} category by {{order}} price",
+ "instantiation_dict": {
+ "product_category": "nutrition bars and drinks",
+ "order": "ascending"
+ },
+ "intent": "List products from nutrition bars and drinks category by ascending price",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/health-household/diet-sports-nutrition/nutrition-bars-drinks.html?product_list_order=price",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/health-household.html?cat=192&product_list_order=price"}]
+ },
+ "intent_template_id": 137
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 353,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List products from {{product_category}} category by {{order}} price",
+ "instantiation_dict": {
+ "product_category": "competitive swimwear",
+ "order": "ascending"
+ },
+ "intent": "List products from competitive swimwear category by ascending price",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/clothing-shoes-jewelry/sport-specific-clothing/competitive-swimwear.html?product_list_order=price",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/clothing-shoes-jewelry.html?cat=149&product_list_order=price"}]
+ },
+ "intent_template_id": 137
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 354,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List products from {{product_category}} category by {{order}} price",
+ "instantiation_dict": {
+ "product_category": "living room furtniture",
+ "order": "descending"
+ },
+ "intent": "List products from living room furtniture category by descending price",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/home-kitchen/furniture/living-room-furniture.html?product_list_order=price&product_list_dir=desc",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=154&product_list_order=price&product_list_dir=desc"}]
+ },
+ "intent_template_id": 137
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 355,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List products from {{product_category}} category by {{order}} price",
+ "instantiation_dict": {
+ "product_category": "kids' bedding",
+ "order": "descending"
+ },
+ "intent": "List products from kids' bedding category by descending price",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/home-kitchen/bedding/kids-bedding.html?product_list_dir=desc",
+ "program_html": [],
+ "url_note": "GOLD in PRED",
+ "or": [{"reference_url": "__SHOPPING__/home-kitchen.html?cat=155&product_list_dir=desc"}]
+ },
+ "intent_template_id": 137
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 356,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
+ "instantiation_dict": {},
+ "intent": "Show the route from SCS CMU in Pittsburgh to the location where the Declaration of Independence and Constitution were signed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Gates and Hillman Centers",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Independence Hall",
+ "Philadelphia"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 49
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 357,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Checkout merge requests requiring my review",
+ "instantiation_dict": {},
+ "intent": "Checkout merge requests requiring my review",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/dashboard/merge_requests?reviewer_username=byteblaze",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 291
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 358,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the {{info}} for order number {{order_number}}.",
+ "instantiation_dict": {
+ "info": "shipping method",
+ "order_number": 187
+ },
+ "intent": "Show me the shipping method for order number 187.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Flat Rate - Fixed"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Flat Rate - Fixed"
+ },
+ "intent_template_id": 206
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 359,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the {{info}} for order number {{order_number}}.",
+ "instantiation_dict": {
+ "info": "order date",
+ "order_number": "148"
+ },
+ "intent": "Show me the order date for order number 148.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "1/29/2023"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1/29/2023"
+ },
+ "intent_template_id": 206
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 360,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the {{info}} for order number {{order_number}}.",
+ "instantiation_dict": {
+ "info": "product names",
+ "order_number": "148"
+ },
+ "intent": "Show me the product names for order number 148.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress)",
+ "Russound 5B45W 4\" Indoor Outdoor Speakers White"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Bornbridge Artificial Spiral Topiary Tree - Indoor / Outdoor Topiary Trees - Artificial Outdoor Plants (2 Pack, 4' Cypress), Russound 5B45W 4\" Indoor Outdoor Speakers White"
+ },
+ "intent_template_id": 206
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 361,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the {{info}} for order number {{order_number}}.",
+ "instantiation_dict": {
+ "info": "order statuses",
+ "order_number": "170 and 189"
+ },
+ "intent": "Show me the order statuses for order number 170 and 189.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": [
+ "170: cancelled",
+ "189: pending"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "170: cancelled, 189: pending"
+ },
+ "intent_template_id": 206
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 362,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Show me the {{info}} for order number {{order_number}}.",
+ "instantiation_dict": {
+ "info": "billing address",
+ "order_number": "00178"
+ },
+ "intent": "Show me the billing address for order number 00178.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "101 S San Mateo Dr",
+ "San Mateo",
+ "California",
+ "94010",
+ "United States"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Emma Lopez, 101 S San Mateo Dr, San Mateo, California, 94010, United States"
+ },
+ "intent_template_id": 206
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 363,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Mellon University",
+ "location/address_2": "Carnegie Music Hall"
+ },
+ "intent": "Measure distance between Carnegie Mellon University and Carnegie Music Hall by walking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["778m |OR| 778 m"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "748m"
+ },
+ "intent_template_id": 58
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 364,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Mellon University",
+ "location/address_2": "UPMC Shadyside"
+ },
+ "intent": "Measure distance between Carnegie Mellon University and UPMC Shadyside by walking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["1.7km |OR| 1.7 km"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1.7km"
+ },
+ "intent_template_id": 58
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 365,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Music Hall",
+ "location/address_2": "UPMC Shadyside"
+ },
+ "intent": "Measure distance between Carnegie Music Hall and UPMC Shadyside by walking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["2.2km |OR| 2.2 km"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "2.2km"
+ },
+ "intent_template_id": 58
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 366,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
+ "instantiation_dict": {
+ "location/address_1": "CVS (closet one)",
+ "location/address_2": "UPMC Shadyside"
+ },
+ "intent": "Measure distance between CVS (closet one) and UPMC Shadyside by walking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["1.2km |OR| 1.2 km"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1.2km"
+ },
+ "intent_template_id": 58
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 367,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Measure distance between {{location/address_1}} and {{location/address_2}} by walking",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Mellon University",
+ "location/address_2": "CVS (closet one)"
+ },
+ "intent": "Measure distance between Carnegie Mellon University and CVS (closet one) by walking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": ["1.4km |OR| 1.4 km"]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "1.4km"
+ },
+ "intent_template_id": 58
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 368,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "find discounted items.",
+ "instantiation_dict": {},
+ "intent": "find discounted items.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no function to show only discount items",
+ "reference_answer_raw_annotation": "There is no function to show only discount items."
+ },
+ "intent_template_id": 188
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 369,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Pull up the description page of {{location}} on Map",
+ "instantiation_dict": {
+ "location": "Carnegie Music Hall"
+ },
+ "intent": "Pull up the description page of Carnegie Music Hall on Map",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Music Hall"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 52
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 370,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Pull up the description page of {{location}} on Map",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University"
+ },
+ "intent": "Pull up the description page of Carnegie Mellon University on Map",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 52
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 371,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Pull up the description page of {{location}} on Map",
+ "instantiation_dict": {
+ "location": "Piada restaurant near Pitt"
+ },
+ "intent": "Pull up the description page of Piada restaurant near Pitt on Map",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Piada Italian Street Food",
+ "Forbes Avenue"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 52
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 372,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Pull up the description page of {{location}} on Map",
+ "instantiation_dict": {
+ "location": "the Costco in Pittsburhg near a river"
+ },
+ "intent": "Pull up the description page of the Costco in Pittsburhg near a river on Map",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Costco",
+ "Waterfront Drive West"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 52
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 373,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Pull up the description page of {{location}} on Map",
+ "instantiation_dict": {
+ "location": "Whole Foods near Carnegie Mellon"
+ },
+ "intent": "Pull up the description page of Whole Foods near Carnegie Mellon on Map",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": null,
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Whole Foods",
+ "East Liberty"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 52
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 374,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Preview the {{name}} theme for my shop",
+ "instantiation_dict": {
+ "name": "Magento Blank"
+ },
+ "intent": "Preview the Magento Blank theme for my shop",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/1",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 266
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 375,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Preview the {{name}} theme for my shop",
+ "instantiation_dict": {
+ "name": "Magento Luma"
+ },
+ "intent": "Preview the Magento Luma theme for my shop",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/admin/system_design_theme/edit/id/3/key/",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 266
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 376,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Summarize customer reviews for {{product}}.",
+ "instantiation_dict": {
+ "product": "Amazon Echo Dot 3rd generation"
+ },
+ "intent": "Summarize customer reviews for Amazon Echo Dot 3rd generation.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no review for this product",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 182
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 377,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the {{space}} around {{location}}",
+ "instantiation_dict": {
+ "location": "CMU ArtPark Lab",
+ "space": "resturants"
+ },
+ "intent": "Find the resturants around CMU ArtPark Lab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__MAP__/search?query=restaurants%20near%20CMU%20ArtPark%20Lab",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 59
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 378,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the {{space}} around {{location}}",
+ "instantiation_dict": {
+ "location": "CMU main campus",
+ "space": "parking"
+ },
+ "intent": "Find the parking around CMU main campus",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__MAP__/search?query=parking%20near%20carnegie%20mellon%20university",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 59
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 379,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the {{space}} around {{location}}",
+ "instantiation_dict": {
+ "location": "CMU main campus",
+ "space": "hotel"
+ },
+ "intent": "Find the hotel around CMU main campus",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__MAP__/search?query=hotels%20near%20carnegie%20mellon%20university",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 59
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 380,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the {{space}} around {{location}}",
+ "instantiation_dict": {
+ "location": "Carnegie Music Hall",
+ "space": "bar"
+ },
+ "intent": "Find the bar around Carnegie Music Hall",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__MAP__/search?query=bars%20near%20Carnegie%20Music%20Hall",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 59
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 381,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the {{space}} around {{location}}",
+ "instantiation_dict": {
+ "location": "Carnegie Music Hall",
+ "space": "hotel"
+ },
+ "intent": "Find the hotel around Carnegie Music Hall",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match"
+ ],
+ "reference_answers": null,
+ "reference_url": "__MAP__/search?query=hotels%20near%20Carnegie%20Music%20Hall",
+ "program_html": [],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 59
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 382,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
+ "instantiation_dict": {},
+ "intent": "I am arriving at Carnegie Mellon University. Find the nearby US Citizenship and Immigration Services and the walking distance to the nearest Social Security Administration from US Citizenship and Immigration Services",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "There is no USCIS nearby",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 781
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 383,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
+ "instantiation_dict": {},
+ "intent": "I am arriving at Pittsburgh Airport. Show me the name of a Hyatt hotel if there is any nearby. Tell me the names of supermarkets that are within 15mins driving from the hotel",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Hyatt Regency Pittsburgh International Airport",
+ "Giant Eagle",
+ "ALDI"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Hyatt Regency Pittsburgh International Airport Giant Eagle, ALDI"
+ },
+ "intent_template_id": 782
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 384,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List the customer names who complain about the quality of EYZUTAK phone cases",
+ "instantiation_dict": {},
+ "intent": "List the customer names who complain about the quality of EYZUTAK phone cases",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Lisa Lee",
+ "Evelyn Kurver",
+ "Amanda",
+ "N Randall"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lisa Lee, Evelyn Kurver, Amanda, N Randall"
+ },
+ "intent_template_id": 666
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 385,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "List the customer names who thinks EYZUTAK phone cases are of good looking",
+ "instantiation_dict": {},
+ "intent": "List the customer names who thinks EYZUTAK phone cases are of good looking",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Lisa Lee",
+ "MH",
+ "Misba009",
+ "Amanda",
+ "N Randall",
+ "Amazon Customer",
+ "Cally",
+ "Bethany Robertson"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Lisa Lee, MH, Misba009, Amanda, N Randall, Amazon Customer, Cally, Bethany Robertson"
+ },
+ "intent_template_id": 666
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 386,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "What is the rating of {{product}}",
+ "instantiation_dict": {
+ "product": "Ugreen lightning to 3.5mm cable"
+ },
+ "intent": "What is the rating of Ugreen lightning to 3.5mm cable. Please round to the nearest whole number",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "65 |OR| 3"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Around 3.1 stars"
+ },
+ "intent_template_id": 1355
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 387,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
+ "instantiation_dict": {
+ "stars": "4 or 5 stars"
+ },
+ "intent": "Who gave 4 or 5 stars for phone cases from EYZUTAK",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "MH",
+ "Misba009",
+ "Amanda",
+ "Amazon Customer",
+ "Cally",
+ "Bethany Robertson"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "MH, Misba009, Amanda, Amazon Customer, Cally, Bethany Robertson, Bethany Robertson"
+ },
+ "intent_template_id": 1356
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 388,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Who gave {{stars}} for phone cases from EYZUTAK",
+ "instantiation_dict": {
+ "stars": "1 or 2 stars"
+ },
+ "intent": "Who gave 1 or 2 stars for phone cases from EYZUTAK",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Evelyn Kurver",
+ "N Randall"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Evelyn Kurver, N Randall"
+ },
+ "intent_template_id": 1356
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 389,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
+ "instantiation_dict": {
+ "mr": "octovisuals page",
+ "repo": "primer/design",
+ "content": "Thanks, working on reviews"
+ },
+ "intent": "Post \"Thanks, working on reviews\" for the merge request related to octovisuals page in primer/design project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/primer/design/-/merge_requests/450",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Thanks, working on reviews"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 348
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 390,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
+ "instantiation_dict": {
+ "mr": "semantic HTML post",
+ "repo": "a11yproject/a11yproject.com",
+ "content": "lgtm"
+ },
+ "intent": "Post \"lgtm\" for the merge request related to semantic HTML post in a11yproject/a11yproject.com project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1531",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "exact_match": "lgtm"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 348
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 391,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
+ "instantiation_dict": {
+ "mr": "focus edge cases",
+ "repo": "a11yproject/a11yproject.com",
+ "content": "close because non reproducible"
+ },
+ "intent": "Post \"close because non reproducible\" for the merge request related to focus edge cases in a11yproject/a11yproject.com project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1265",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "exact_match": "close because non reproducible"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 348
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 392,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
+ "instantiation_dict": {
+ "mr": "color ulitity",
+ "repo": "a11yproject.com",
+ "content": "Good idea"
+ },
+ "intent": "Post \"Good idea\" for the merge request related to color ulitity in a11yproject.com project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1071",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "exact_match": "Good idea"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 348
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 393,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Post \"{{content}}\" for the merge request related to {{mr}} in {{repo}} project",
+ "instantiation_dict": {
+ "mr": "fixing the broken links",
+ "repo": "byteblaze/empathy-prompts",
+ "content": "lgtm"
+ },
+ "intent": "Post \"lgtm\" for the merge request related to fixing the broken links in byteblaze/empathy-prompts project",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/empathy-prompts/-/merge_requests/19",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "exact_match": "lgtm"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 348
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 394,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "2019-nCov"
+ },
+ "intent": "Fork 2019-nCov.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/2019-nCov",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "2019-nCov"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 395,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "the Pytorch GAN repo with most stars"
+ },
+ "intent": "Fork the Pytorch GAN repo with most stars.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/PyTorch-GAN",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Pytorch-GAN"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 396,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "ChatGPT"
+ },
+ "intent": "Fork ChatGPT.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/ChatGPT",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "ChatGPT"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 397,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "MetaSeq"
+ },
+ "intent": "Fork MetaSeq.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/metaseq",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "metaseq"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 398,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "all source repos from Akilesh Kannan"
+ },
+ "intent": "Fork all source repos from Akilesh Kannan",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/SimCache",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "SimCache"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/dots",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "dots"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/CacheEval",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "CacheEval"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/nvidia-patch",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "404"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/viewgrades-scraper",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "404"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 399,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Change my reddit bio to \"{{content}}\"",
+ "instantiation_dict": {
+ "content": "I am a robot"
+ },
+ "intent": "Change my reddit bio to \"I am a robot\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/user/MarvelsGrantMan136",
+ "locator": "document.querySelector(\".user-bio__biography\").outerText",
+ "required_contents": {
+ "exact_match": "I am a robot"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 6
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 400,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Change my reddit bio to \"{{content}}\"",
+ "instantiation_dict": {
+ "content": "Pro Python Developer with 20 years of Experience"
+ },
+ "intent": "Change my reddit bio to \"Pro Python Developer with 20 years of Experience\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/user/MarvelsGrantMan136",
+ "locator": "document.querySelector(\".user-bio__biography\").outerText",
+ "required_contents": {
+ "exact_match": "Pro Python Developer with 20 years of Experience"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 6
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 401,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Change my reddit bio to \"{{content}}\"",
+ "instantiation_dict": {
+ "content": "Seeking SDE positions"
+ },
+ "intent": "Change my reddit bio to \"Seeking SDE positions\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/user/MarvelsGrantMan136",
+ "locator": "document.querySelector(\".user-bio__biography\").outerText",
+ "required_contents": {
+ "exact_match": "Seeking SDE positions"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 6
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 402,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Change my reddit bio to \"{{content}}\"",
+ "instantiation_dict": {
+ "content": "Freelance Web Developer"
+ },
+ "intent": "Change my reddit bio to \"Freelance Web Developer\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/user/MarvelsGrantMan136",
+ "locator": "document.querySelector(\".user-bio__biography\").outerText",
+ "required_contents": {
+ "exact_match": "Freelance Web Developer"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 6
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 403,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Change my reddit bio to \"{{content}}\"",
+ "instantiation_dict": {
+ "content": "Awesome Prompt Artist"
+ },
+ "intent": "Change my reddit bio to \"Awesome Prompt Artist\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/user/MarvelsGrantMan136",
+ "locator": "document.querySelector(\".user-bio__biography\").outerText",
+ "required_contents": {
+ "exact_match": "Awesome Prompt Artist"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 6
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 404,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Upvote the newest post in {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "subreddit": "books"
+ },
+ "intent": "Upvote the newest post in books subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/books/124260/adults-reading-to-each-other-out-loud",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 22
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 405,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Upvote the newest post in {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "subreddit": "DIY"
+ },
+ "intent": "Upvote the newest post in DIY subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/DIY/119019/how-can-i-bring-an-hdmi-cable-from-my-pc-downstairs-to-my-tv",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 22
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 406,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Upvote the newest post in {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "subreddit": "future technology"
+ },
+ "intent": "Upvote the newest post in future technology subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/Futurology/119517/openai-ceo-it-s-not-funny-that-i-m-afraid-of-the-ai-we-re",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 22
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 407,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Upvote the newest post in {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "subreddit": "deeplearning"
+ },
+ "intent": "Upvote the newest post in deeplearning subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/deeplearning/124993/meta-s-llama-weights-leaked-on-torrent-and-the-best-thing",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 22
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 408,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Upvote the newest post in {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "subreddit": "explain like im 5"
+ },
+ "intent": "Upvote the newest post in explain like im 5 subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/explainlikeimfive/39244/eli5-how-does-pinching-a-ribbon-and-sliding-your-finger",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 22
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 409,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
+ "geolocation": null,
+ "intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
+ "instantiation_dict": {
+ "position_description": "the manager of the website",
+ "content_description": "thanks! I am a big fan of your website."
+ },
+ "intent": "Reply to the manager of the website in this post with \"thanks! I am a big fan of your website.\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/books/59421/-/comment/1235250",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "thanks! I am a big fan of your website."
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 23
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 410,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/singularity/69404/this-is-how-chatgpt-sees-itself",
+ "geolocation": null,
+ "intent_template": "Reply to {{position_description}} in this post with \"{{content_description}}\"",
+ "instantiation_dict": {
+ "position_description": "the first reply",
+ "content_description": "don't panic"
+ },
+ "intent": "Reply to the first reply in this post with \"don't panic\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/singularity/69404/-/comment/1042264",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "don't panic"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 23
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 411,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make the LICENSE of {{repo}} to MIT license.",
+ "instantiation_dict": {
+ "repo": "byteblaze/cloud-to-butt"
+ },
+ "intent": "Make the LICENSE of byteblaze/cloud-to-butt to MIT license.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/cloud-to-butt/-/blob/master/LICENSE.txt",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "MIT license"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 355
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 412,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make the LICENSE of {{repo}} to Apache License",
+ "instantiation_dict": {
+ "repo": "byteblaze/accessible-html-content-patterns"
+ },
+ "intent": "Make the LICENSE of byteblaze/accessible-html-content-patterns to Apache License",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/accessible-html-content-patterns/-/blob/main/LICENSE",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Apache License",
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 355
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 413,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make the LICENSE of {{repo}} to one that mandates all copies and derivative works to be under the same license",
+ "instantiation_dict": {
+ "repo": "byteblaze/a11y-syntax-highlighting"
+ },
+ "intent": "Make the LICENSE of byteblaze/a11y-syntax-highlighting to one that mandates all copies and derivative works to be under the same license",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/blob/main/LICENSE",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "GENERAL PUBLIC LICENSE"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 355
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 414,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make the LICENSE of {{repo}} to MIT license.",
+ "instantiation_dict": {
+ "repo": "byteblaze/dotfiles"
+ },
+ "intent": "Make the LICENSE of byteblaze/dotfiles to MIT license.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/blob/main/LICENSE",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "MIT license",
+ "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 355
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 415,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "instantiation_dict": {
+ "topic": "verification functions"
+ },
+ "intent": "Go to the merge request on verification functions I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/a11y-webring.club/-/merge_requests/40",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "must_include": ["@davepgreene"]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 360
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 416,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "instantiation_dict": {
+ "topic": "wcag"
+ },
+ "intent": "Go to the merge request on wcag I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1270",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "must_include": ["Thank you"]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 360
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 417,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Go to the merge request on {{topic}} I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "instantiation_dict": {
+ "topic": "404 link"
+ },
+ "intent": "Go to the merge request on 404 link I have to review, find if the author of the merge request responded at the end, and reply \"Thank you\" if he did. Otherwise remind him with a simple @.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests/1485",
+ "locator": "document.querySelector('[id=\"notes-list\"').lastElementChild.querySelector('.timeline-discussion-body').outerText",
+ "required_contents": {
+ "must_include": ["@Roshanjossey"]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 360
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 418,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set my gitlab status as {{status}}.",
+ "instantiation_dict": {
+ "status": "Busy"
+ },
+ "intent": "Set my gitlab status as Busy.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.cover-title').textContent",
+ "required_contents": {
+ "must_include": ["Busy"]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 361
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 419,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set my gitlab status as {{status}}.",
+ "instantiation_dict": {
+ "status": "Enjoying life"
+ },
+ "intent": "Set my gitlab status as Enjoying life.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.cover-status').lastChild.textContent",
+ "required_contents": {
+ "exact_match": "Enjoying life"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 361
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 420,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set my gitlab status as {{status}}.",
+ "instantiation_dict": {
+ "status": "Playing Badminton"
+ },
+ "intent": "Set my gitlab status as Playing Badminton.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.cover-status').lastChild.textContent",
+ "required_contents": {
+ "exact_match": "Playing Badminton"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 361
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 421,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set my gitlab status as {{status}}.",
+ "instantiation_dict": {
+ "status": "Resting due to leg injury"
+ },
+ "intent": "Set my gitlab status as Resting due to leg injury.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.cover-status').lastChild.textContent",
+ "required_contents": {
+ "exact_match": "Resting due to leg injury"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 361
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 422,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set my gitlab status as {{status}}.",
+ "instantiation_dict": {
+ "status": "Out of Office"
+ },
+ "intent": "Set my gitlab status as Out of Office.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.cover-status').lastChild.textContent",
+ "required_contents": {
+ "exact_match": "Out of Office"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 361
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 423,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Mark all {{brand}} shirts on sale",
+ "instantiation_dict": {
+ "brand": "Hollister"
+ },
+ "intent": "Mark all Hollister shirts on sale",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/126/",
+ "locator": "document.querySelector('input[name=\"product[sale]\"]').value",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 237
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 424,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the place where Mr. Rogers was filmed"
+ },
+ "intent": "Find the page of the place where Mr. Rogers was filmed on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Pittsburgh"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 425,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the longest bridge in the Western hemisphere"
+ },
+ "intent": "Find the page of the longest bridge in the Western hemisphere on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Mackinac Bridge"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 426,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the place in Pennsylvania where a plane crashed during the September 11th attacks"
+ },
+ "intent": "Find the page of the place in Pennsylvania where a plane crashed during the September 11th attacks on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Somerset County"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 427,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the university that has most Turning Award winners"
+ },
+ "intent": "Find the page of the university that has most Turning Award winners on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Massachusetts Institute of Technology"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 428,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the undergrad college of the person who developed the Nash equilibrium"
+ },
+ "intent": "Find the page of the undergrad college of the person who developed the Nash equilibrium on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 429,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the colleges where The Chair was filmed in Pittsburgh"
+ },
+ "intent": "Find the page of the colleges where The Chair was filmed in Pittsburgh on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Chatham University"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 430,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the page of {{description}} on the map.",
+ "instantiation_dict": {
+ "description": "the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh"
+ },
+ "intent": "Find the page of the college(s) where The Chair was filmed in Pennsylvania other than the ones in Pittsburgh on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sidebar_content\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Washington & Jefferson College"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 371
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 431,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/tall-pink-taper-candles-4-piece-orange-colored-tapered-candles-gradient-candles-10-6-inches-tall-tie-dye-candle-set-large-dripless-long-burning-candlesticks-two-color-taper-candles-candlesticks.html |AND| __SHOPPING__/spaas-white-taper-candles-4-pack-10-inch-tall-candles-scent-free-premium-wax-candle-sticks-8-hour-long-burning-white-candlesticks-for-home-decoration-wedding-holiday-and-parties.html |AND| __SHOPPING__/white-starfish-wall-candle-sconces-set-of-2-beach-decor-ocean-themed-wall-mount-candleholders-nautical-style-beach-bathroom-decor-coastal-farmhouse-seashell-candle-holders.html",
+ "geolocation": null,
+ "intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "instantiation_dict": {},
+ "intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/checkout/cart",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "SPAAS White Taper Candles - 4 Pack |OR| 10 Inch Tall Candles, Scent-Free Premium Wax Candle Sticks |OR| 8 Hour Long Burning White Candlesticks for Home Decoration, Wedding, Holiday and Parties"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 145
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 432,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/ciclon-energy-drink-regular-24-cans-8-3oz.html |AND| __SHOPPING__/v8-energy-healthy-energy-drink-steady-energy-from-black-and-green-tea-pomegranate-blueberry-8-ounce-can-pack-of-24.html",
+ "geolocation": null,
+ "intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "instantiation_dict": {},
+ "intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/checkout/cart",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "V8 +Energy, Healthy Energy Drink, Steady Energy from Black and Green Tea, Pomegranate Blueberry, 8 Ounce Can ,Pack of 24"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 145
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 433,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/tazrigo-5pcs-white-dental-resin-brush-pens-dental-shaping-silicone-tooth-tool.html |AND| __SHOPPING__/stylus-pens-for-touch-screens-2-pcs-universal-stylus-2-in-1-2022-updated-touch-screen-pens-for-all-touch-screens-cell-phones-tablets-laptops-with-6-replacement-tips-4-discstips-2-fiber-tips.html",
+ "geolocation": null,
+ "intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "instantiation_dict": {},
+ "intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/checkout/cart",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tazrigo 5pcs White Dental Resin Brush Pens Dental Shaping Silicone Tooth Tool"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 145
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 434,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/3-pairs-ruffle-socks-lace-ankle-socks-for-girls-frilly-socks-women-decorative.html |AND| __SHOPPING__/viviki-women-glitter-socks-ultrathin-transparent-tulle-lace-socks-no-show-ankle-crew-socks-3-pack.html",
+ "geolocation": null,
+ "intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "instantiation_dict": {},
+ "intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/checkout/cart",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "VIVIKI Women Glitter Socks Ultrathin Transparent Tulle Lace Socks - No Show Ankle Crew Socks 3 Pack"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 145
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 435,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/35-ft-hdmi-cable-gearit-pro-series-hdmi-cable-35-feet-high-speed-ethernet-4k-resolution-3d-video-and-arc-audio-return-channel-hdmi-cable-white.html |AND| __SHOPPING__/dp-to-hdmi-cable-6ft-2-pack-fosmon-gold-plated-displayport-to-hdmi-cable-1080p-full-hd-for-pcs-to-hdtv-monitor-projector-with-hdmi-port.html",
+ "geolocation": null,
+ "intent_template": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "instantiation_dict": {},
+ "intent": "Add the product with the lowest per unit price from my open tabs to the shopping cart",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/checkout/cart",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "DP to HDMI Cable 6FT (2 Pack), Fosmon Gold Plated Displayport to HDMI Cable 1080p Full HD for PCs to HDTV, Monitor, Projector with HDMI Port"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 145
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 436,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
+ "instantiation_dict": {
+ "product": "a mattress foundation",
+ "time": "around Feb or March 2023"
+ },
+ "intent": "I previously ordered some a mattress foundation around Feb or March 2023 and later cancelled. Can you reorder it for me?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B07DFJ5XKH"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 156
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 437,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
+ "instantiation_dict": {
+ "product": "a table lamp",
+ "time": "in May 2023"
+ },
+ "intent": "I previously ordered some a table lamp in May 2023 and later cancelled. Can you reorder it for me?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B072XS3F6W"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 156
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 438,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
+ "instantiation_dict": {
+ "product": "a TV stand",
+ "time": "sometime around sep 2022"
+ },
+ "intent": "I previously ordered some a TV stand sometime around sep 2022 and later cancelled. Can you reorder it for me?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B08PVHRRB7"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 156
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 439,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
+ "instantiation_dict": {
+ "product": "a cat t-shirt",
+ "time": "during 2022"
+ },
+ "intent": "I previously ordered some a cat t-shirt during 2022 and later cancelled. Can you reorder it for me?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B0844BWS76"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 156
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 440,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I previously ordered some {{product}} {{time}} and later cancelled. Can you reorder it for me?",
+ "instantiation_dict": {
+ "product": "a make up removal kit",
+ "time": "during summer 2022"
+ },
+ "intent": "I previously ordered some a make up removal kit during summer 2022 and later cancelled. Can you reorder it for me?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B0738JQG6Q"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 156
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 441,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/gimmiethat.space",
+ "geolocation": null,
+ "intent_template": "Update the project site's title to \"{{title}}\"",
+ "instantiation_dict": {
+ "title": "GIVE ME SPACE"
+ },
+ "intent": "Update the project site's title to \"GIVE ME SPACE\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "GIVE ME SPACE "
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 308
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 442,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/gimmiethat.space",
+ "geolocation": null,
+ "intent_template": "Update the project site's title to \"{{title}}\"",
+ "instantiation_dict": {
+ "title": "Welcome to my site"
+ },
+ "intent": "Update the project site's title to \"Welcome to my site\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Welcome to my site "
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 308
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 443,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/gimmiethat.space",
+ "geolocation": null,
+ "intent_template": "Update the project site's title to \"{{title}}\"",
+ "instantiation_dict": {
+ "title": "Not an interesting site"
+ },
+ "intent": "Update the project site's title to \"Not an interesting site\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Not an interesting site "
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 308
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 444,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/gimmiethat.space",
+ "geolocation": null,
+ "intent_template": "Update the project site's title to \"{{title}}\"",
+ "instantiation_dict": {
+ "title": "Title Wanted"
+ },
+ "intent": "Update the project site's title to \"Title Wanted\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Title Wanted "
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 308
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 445,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/gimmiethat.space",
+ "geolocation": null,
+ "intent_template": "Update the project site's title to \"{{title}}\"",
+ "instantiation_dict": {
+ "title": "Hello"
+ },
+ "intent": "Update the project site's title to \"Hello\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/index.html",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Hello "
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 308
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 446,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
+ "instantiation_dict": {
+ "repo": "a11yproject",
+ "issue": 404,
+ "account": "Roshanjossey"
+ },
+ "intent": "Assign the issue regarding 404 in a11yproject to Roshanjossey.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Roshanjossey",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "404s, bad host, timeouts, bad urls for URLs linked from website"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 999
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 447,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
+ "instantiation_dict": {
+ "repo": "a11y-webring.club",
+ "issue": "linking to an accessibility statement",
+ "account": "Rohan"
+ },
+ "intent": "Assign the issue regarding linking to an accessibility statement in a11y-webring.club to Rohan.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=Seirdy",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "linking to an accessibility statement"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 999
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 448,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "set the homepage URL on my GitLab profile to {{url}}",
+ "instantiation_dict": {
+ "url": "https://egg.tart.com"
+ },
+ "intent": "set the homepage URL on my GitLab profile to https://egg.tart.com",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
+ "required_contents": {
+ "exact_match": "egg.tart.com"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 331
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 449,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "set the homepage URL on my GitLab profile to {{url}}",
+ "instantiation_dict": {
+ "url": "https://helloworld.xyz"
+ },
+ "intent": "set the homepage URL on my GitLab profile to https://helloworld.xyz",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
+ "required_contents": {
+ "exact_match": "helloworld.xyz"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 331
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 450,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "set the homepage URL on my GitLab profile to {{url}}",
+ "instantiation_dict": {
+ "url": "a11yproject.contributor.me"
+ },
+ "intent": "set the homepage URL on my GitLab profile to a11yproject.contributor.me",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
+ "required_contents": {
+ "exact_match": "a11yproject.contributor.me"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 331
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 451,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "set the homepage URL on my GitLab profile to {{url}}",
+ "instantiation_dict": {
+ "url": "www.byteblaze.com"
+ },
+ "intent": "set the homepage URL on my GitLab profile to www.byteblaze.com",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
+ "required_contents": {
+ "exact_match": "www.byteblaze.com"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 331
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 452,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "set the homepage URL on my GitLab profile to {{url}}",
+ "instantiation_dict": {
+ "url": "byteblaze.github.io"
+ },
+ "intent": "set the homepage URL on my GitLab profile to byteblaze.github.io",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze",
+ "locator": "document.querySelector('.profile-header [itemprop=\"url\"]').outerText",
+ "required_contents": {
+ "exact_match": "byteblaze.github.io"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 331
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 453,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
+ "instantiation_dict": {
+ "product": "Teton pullover hoodie"
+ },
+ "intent": "Disable Teton pullover hoodie from the site, they are facing some quality issues.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/78/",
+ "locator": "document.querySelector('[name=\"product[status]\"').value",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 242
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 454,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
+ "instantiation_dict": {
+ "product": "Ryker Tee Crew Neck"
+ },
+ "intent": "Disable Ryker Tee Crew Neck from the site, they are facing some quality issues.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/478/",
+ "locator": "document.querySelector('[name=\"product[status]\"').value",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 242
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 455,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
+ "instantiation_dict": {
+ "product": "lHelios Endurance Tank"
+ },
+ "intent": "Disable lHelios Endurance Tank from the site, they are facing some quality issues.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/676/",
+ "locator": "document.querySelector('[name=\"product[status]\"').value",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 242
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 456,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
+ "instantiation_dict": {
+ "product": "Cora Pant"
+ },
+ "intent": "Disable Cora Pant from the site, they are facing some quality issues.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1840/",
+ "locator": "document.querySelector('[name=\"product[status]\"').value",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 242
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 457,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Disable {{product}} from the site, they are facing some quality issues.",
+ "instantiation_dict": {
+ "product": "Karmen yoga pants"
+ },
+ "intent": "Disable Karmen yoga pants from the site, they are facing some quality issues.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1819/",
+ "locator": "document.querySelector('[name=\"product[status]\"').value",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 242
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 458,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "$5",
+ "action": "Reduce"
+ },
+ "intent": "Reduce the price of this product by $5",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "27.00"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 459,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "10%",
+ "action": "Reduce"
+ },
+ "intent": "Reduce the price of this product by 10%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/237/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "62.10"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 460,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "15%",
+ "action": "Reduce"
+ },
+ "intent": "Reduce the price of this product by 15%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/418/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "38.25"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 461,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "$11.5",
+ "action": "Increase"
+ },
+ "intent": "Increase the price of this product by $11.5",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/721/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "29.50"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 462,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "10%",
+ "action": "Increase"
+ },
+ "intent": "Increase the price of this product by 10%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/710/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "19.80"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 463,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of this product by {{amount}}",
+ "instantiation_dict": {
+ "amount": "15%",
+ "action": "Increase"
+ },
+ "intent": "Increase the price of this product by 15%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/996/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "36.80"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 247
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 464,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update the product description of {{product}} to highlight the real user positive reviews by quoting the comments",
+ "instantiation_dict": {
+ "product": "Antonia Racer Tank"
+ },
+ "intent": "Update the product description of Antonia Racer Tank to highlight the real user positive reviews by quoting the comments",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/../antonia-racer-tank.html",
+ "locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
+ "required_contents": {
+ "must_include": [
+ "This is in regular rotation at the gym",
+ "Its colorful and looks kinda cute under my exercise tanks",
+ "it's very stylish for yoga or something else low impact"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 251
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 465,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add {{product}} to my wish list",
+ "instantiation_dict": {
+ "product": "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
+ },
+ "intent": "Add Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count to my wish list",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Tide PODS Spring Meadow Scent HE Turbo Laundry Detergent Pacs, 81 Count"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 186
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 466,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add {{product}} to my wish list",
+ "instantiation_dict": {
+ "product": "2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
+ },
+ "intent": "Add 2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts to my wish list",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "2 Hawaiian Bamboo Orchid Roots #zc50 - by Discount Hawaiian Gifts"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 186
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 467,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add {{product}} to my wish list",
+ "instantiation_dict": {
+ "product": "HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
+ },
+ "intent": "Add HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits to my wish list",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "HONGJ Hawaiian Beach Outfits Set for Mens, Summer Tropical Tree Printed Relaxed-fit Hawaii Shirts Shorts 2 Piece Suits"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 186
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 468,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add {{product}} to my wish list",
+ "instantiation_dict": {
+ "product": "DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
+ },
+ "intent": "Add DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit to my wish list",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "DkRgVNY Lace Spcling Lingerie Womens Sexy Hollow Out Underwear Bodysuit One Piece Snap Crotch Clubwear Teddy Bodysuit"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 186
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 469,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add {{product}} to my wish list",
+ "instantiation_dict": {
+ "product": "Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
+ },
+ "intent": "Add Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes to my wish list",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Light Blue Simple Summer New Low Heels Slippers for Women Fashion Chunky Heels Pointed Toe Wine Glasses Sandals Comfortable Walking Shoes Ladies All-Match Sexy Party Shoes"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 186
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 470,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Cancel order {{id}}",
+ "instantiation_dict": {
+ "id": "302"
+ },
+ "intent": "Cancel order 302",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
+ "locator": "document.querySelector(\"#order_status\").outerText",
+ "required_contents": {
+ "exact_match": "Canceled"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 257
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 471,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Cancel order {{id}}",
+ "instantiation_dict": {
+ "id": "307"
+ },
+ "intent": "Cancel order 307",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/307/",
+ "locator": "document.querySelector(\"#order_status\").outerText",
+ "required_contents": {
+ "exact_match": "Canceled"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 257
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 472,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Cancel order {{id}}",
+ "instantiation_dict": {
+ "id": "299"
+ },
+ "intent": "Cancel order 299",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/299/",
+ "locator": "document.querySelector(\"#order_status\").outerText",
+ "required_contents": {
+ "exact_match": "Canceled"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 257
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 473,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Cancel order {{id}}",
+ "instantiation_dict": {
+ "id": "301"
+ },
+ "intent": "Cancel order 301",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/301/",
+ "locator": "document.querySelector(\"#order_status\").outerText",
+ "required_contents": {
+ "exact_match": "Canceled"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 257
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 474,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Cancel order {{id}}",
+ "instantiation_dict": {
+ "id": "305"
+ },
+ "intent": "Cancel order 305",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/305/",
+ "locator": "document.querySelector(\"#order_status\").outerText",
+ "required_contents": {
+ "exact_match": "Canceled"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 257
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 475,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set up a new, empty repository with the name {{project_name}}?",
+ "instantiation_dict": {
+ "project_name": "chatgpt_plugin"
+ },
+ "intent": "Set up a new, empty repository with the name chatgpt_plugin?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/chatgpt_plugin",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "chatgpt_plugin"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 292
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 476,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set up a new, empty repository with the name {{project_name}}?",
+ "instantiation_dict": {
+ "project_name": "awesome_llm_reading"
+ },
+ "intent": "Set up a new, empty repository with the name awesome_llm_reading?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/awesome_llm_reading",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "awesome_llm_reading"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 292
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 477,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set up a new, empty repository with the name {{project_name}}?",
+ "instantiation_dict": {
+ "project_name": "awesome_program_aided_reasoning"
+ },
+ "intent": "Set up a new, empty repository with the name awesome_program_aided_reasoning?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/awesome_program_aided_reasoning",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "awesome_program_aided_reasoning"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 292
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 478,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set up a new, empty repository with the name {{project_name}}?",
+ "instantiation_dict": {
+ "project_name": "webagent"
+ },
+ "intent": "Set up a new, empty repository with the name webagent?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/webagent",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "webagent"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 292
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 479,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Set up a new, empty repository with the name {{project_name}}?",
+ "instantiation_dict": {
+ "project_name": "awesome_webagent"
+ },
+ "intent": "Set up a new, empty repository with the name awesome_webagent?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/awesome_webagent",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "awesome_webagent"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 292
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 480,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}}",
+ "instantiation_dict": {
+ "collaborator_account_list": "yjlou",
+ "repo": "solarized-prism-theme"
+ },
+ "intent": "Invite yjlou as collaborator to solarized-prism-theme",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/solarized-prism-theme/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "yjlou"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 293
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 481,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "instantiation_dict": {
+ "name": "Abishek"
+ },
+ "intent": "Abishek wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 294
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 482,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "instantiation_dict": {
+ "name": "yjlou"
+ },
+ "intent": "yjlou wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 294
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 483,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "instantiation_dict": {
+ "name": "Koushik"
+ },
+ "intent": "Koushik wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'koush')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 294
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 484,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "instantiation_dict": {
+ "name": "Jakub Klinkovsk\u00fd"
+ },
+ "intent": "Jakub Klinkovsk\u00fd wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 294
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 485,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "{{name}} wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "instantiation_dict": {
+ "name": "Vinta"
+ },
+ "intent": "Vinta wants to check my dotfile configurations. Please invite him to the repo as a guest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'vinta')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 294
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 486,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
+ "instantiation_dict": {
+ "old-heading": "404 Not Found",
+ "heading": "Bruh bro you clicked the wrong page"
+ },
+ "intent": "Change the page title of \"404 Not Found\" page on my site to \"Bruh bro you clicked the wrong page\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/1/",
+ "locator": "document.querySelector('input[name=\"title\"').value",
+ "required_contents": {
+ "exact_match": "Bruh bro you clicked the wrong page"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 275
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 487,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
+ "instantiation_dict": {
+ "old-heading": "Enable Cookies",
+ "heading": "Cookie monster coming to your place"
+ },
+ "intent": "Change the page title of \"Enable Cookies\" page on my site to \"Cookie monster coming to your place\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/3/",
+ "locator": "document.querySelector('input[name=\"title\"').value",
+ "required_contents": {
+ "exact_match": "Cookie monster coming to your place"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 275
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 488,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
+ "instantiation_dict": {
+ "old-heading": "Home Page",
+ "heading": "This is the home page!! Leave here!!"
+ },
+ "intent": "Change the page title of \"Home Page\" page on my site to \"This is the home page!! Leave here!!\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/2/",
+ "locator": "document.querySelector('input[name=\"title\"').value",
+ "required_contents": {
+ "exact_match": "This is the home page!! Leave here!!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 275
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 489,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
+ "instantiation_dict": {
+ "old-heading": "Privacy Policy",
+ "heading": "No privacy policy is needed in this dystopian world"
+ },
+ "intent": "Change the page title of \"Privacy Policy\" page on my site to \"No privacy policy is needed in this dystopian world\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/4/",
+ "locator": "document.querySelector('input[name=\"title\"').value",
+ "required_contents": {
+ "exact_match": "No privacy policy is needed in this dystopian world"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 275
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 490,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Change the page title of \"{{old-heading}}\" page on my site to \"{{heading}}\".",
+ "instantiation_dict": {
+ "old-heading": "About us",
+ "heading": "Secret"
+ },
+ "intent": "Change the page title of \"About us\" page on my site to \"Secret\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/cms/page/edit/page_id/5/",
+ "locator": "document.querySelector('input[name=\"title\"').value",
+ "required_contents": {
+ "exact_match": "Secret"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 275
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 491,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
+ "instantiation_dict": {
+ "name": "Sarah Miller",
+ "message": "the order is ready to be shipped soon!"
+ },
+ "intent": "Notify Sarah Miller in their most recent pending order with message \"the order is ready to be shipped soon!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "System message: We cannot add order history."
+ },
+ "intent_template_id": 280
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 492,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
+ "instantiation_dict": {
+ "name": "Jane Doe",
+ "message": "sorry we are out of stock, please reorder"
+ },
+ "intent": "Notify Jane Doe in their most recent pending order with message \"sorry we are out of stock, please reorder\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/302/",
+ "locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
+ "required_contents": {
+ "exact_match": "sorry we are out of stock, please reorder"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 280
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 493,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
+ "instantiation_dict": {
+ "name": "Grace Nguyen",
+ "message": "sorry we are bankrupt, please contact our customer service for refund"
+ },
+ "intent": "Notify Grace Nguyen in their most recent pending order with message \"sorry we are bankrupt, please contact our customer service for refund\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/307/",
+ "locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
+ "required_contents": {
+ "exact_match": "sorry we are bankrupt, please contact our customer service for refund"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 280
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 494,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
+ "instantiation_dict": {
+ "name": "Alex Thomas",
+ "message": "Yo, your order will be shipped soon!"
+ },
+ "intent": "Notify Alex Thomas in their most recent pending order with message \"Yo, your order will be shipped soon!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/304/",
+ "locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
+ "required_contents": {
+ "exact_match": "Yo, your order will be shipped soon!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 280
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 495,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Notify {{name}} in their most recent pending order with message \"{{message}}\"",
+ "instantiation_dict": {
+ "name": "Lily Potter",
+ "message": "Thanks, your order is ready to be shipped!"
+ },
+ "intent": "Notify Lily Potter in their most recent pending order with message \"Thanks, your order is ready to be shipped!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/303/",
+ "locator": "document.querySelector(\"#order_history_block\").querySelector(\".note-list\").firstElementChild.querySelector(\".note-list-comment\").outerText",
+ "required_contents": {
+ "exact_match": "Thanks, your order is ready to be shipped!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 280
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 496,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
+ "instantiation_dict": {
+ "tracking": "8974568499",
+ "order": "299",
+ "service": "Federal Express"
+ },
+ "intent": "Update order #299 with the Federal Express tracking number 8974568499",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/299/active_tab/order_shipments/",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tracking number 8974568499 for Federal Express assigned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 284
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 497,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
+ "instantiation_dict": {
+ "tracking": "24353446464",
+ "order": "307",
+ "service": "DHL"
+ },
+ "intent": "Update order #307 with the DHL tracking number 24353446464",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/307/active_tab/order_shipments/",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tracking number 24353446464 for DHL assigned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 284
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 498,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
+ "instantiation_dict": {
+ "tracking": "55591023930",
+ "order": "306",
+ "service": "UPS"
+ },
+ "intent": "Update order #306 with the UPS tracking number 55591023930",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/306/active_tab/order_shipments/",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tracking number 55591023930 for United Parcel Service assigned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 284
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 499,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
+ "instantiation_dict": {
+ "tracking": "13849373987",
+ "order": "304",
+ "service": "USPS"
+ },
+ "intent": "Update order #304 with the USPS tracking number 13849373987",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/304/active_tab/order_shipments/",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tracking number 13849373987 for United States Postal Service assigned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 284
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 500,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update order #{{order}} with the {{service}} tracking number {{tracking}}",
+ "instantiation_dict": {
+ "tracking": "239028439840",
+ "order": "301",
+ "service": "DHL"
+ },
+ "intent": "Update order #301 with the DHL tracking number 239028439840",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/commentsHistory/order_id/301/active_tab/order_shipments/",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Tracking number 239028439840 for DHL assigned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 284
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 501,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Make all {{product}} as out of stock",
+ "instantiation_dict": {
+ "product": "Taurus Elements Shell"
+ },
+ "intent": "Make all Taurus Elements Shell as out of stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/350/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 287
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 502,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Make all {{product}} as out of stock",
+ "instantiation_dict": {
+ "product": "Gobi HeatTec Tee"
+ },
+ "intent": "Make all Gobi HeatTec Tee as out of stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/446/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 287
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 503,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Make all {{product}} as out of stock",
+ "instantiation_dict": {
+ "product": "rocco gym tank"
+ },
+ "intent": "Make all rocco gym tank as out of stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/682/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 287
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 504,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Make all {{product}} as out of stock",
+ "instantiation_dict": {
+ "product": "Selene yoga hoodie"
+ },
+ "intent": "Make all Selene yoga hoodie as out of stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1108/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 287
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 505,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Make all {{product}} as out of stock",
+ "instantiation_dict": {
+ "product": "Aeno capri"
+ },
+ "intent": "Make all Aeno capri as out of stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1861/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 287
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 506,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
+ "instantiation_dict": {
+ "product_category": "meat substitute",
+ "dollar_value": "between 100 and 200"
+ },
+ "intent": "Buy the highest rated product from the meat substitute category within a budget between 100 and 200.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B01CTR3DLE"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 172
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 507,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
+ "instantiation_dict": {
+ "product_category": "Ceiling light",
+ "dollar_value": "above 1000"
+ },
+ "intent": "Buy the highest rated product from the Ceiling light category within a budget above 1000.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B07BVL3P1V"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 172
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 508,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
+ "instantiation_dict": {
+ "product_category": "NS switch pouch",
+ "dollar_value": "under 60"
+ },
+ "intent": "Buy the highest rated product from the NS switch pouch category within a budget under 60.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B07116LGP6"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 172
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 509,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the best rating product from \"{{category}}\" category with at least 5 reviews and the product is least expensive",
+ "instantiation_dict": {
+ "category": "Men's shoe"
+ },
+ "intent": "Buy the best rating product from \"Men's shoe\" category with at least 5 reviews and the product is least expensive",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B01J4MM3KO"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 216
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 510,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the best rating product from \"{{category}}\" category with at least 5 reviews and the product is least expensive",
+ "instantiation_dict": {
+ "category": "Home Audio Speaker"
+ },
+ "intent": "Buy the best rating product from \"Home Audio Speaker\" category with at least 5 reviews and the product is least expensive",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "func:shopping_get_latest_order_url()",
+ "locator": "document.querySelector(\".order-details-items.ordered\").outerText",
+ "required_contents": {
+ "must_include": [
+ "B002R5ABIW"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 216
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 511,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add a {{product}} to my wish list.",
+ "instantiation_dict": {
+ "product": "laundry detergent"
+ },
+ "intent": "Add a laundry detergent to my wish list.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "laundry",
+ "detergent"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 189
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 512,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add a {{product}} to my wish list.",
+ "instantiation_dict": {
+ "product": "toothpaste"
+ },
+ "intent": "Add a toothpaste to my wish list.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "toothpaste"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 189
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 513,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add a {{product}} to my wish list.",
+ "instantiation_dict": {
+ "product": "chair"
+ },
+ "intent": "Add a chair to my wish list.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "chair"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 189
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 514,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add a {{product}} to my wish list.",
+ "instantiation_dict": {
+ "product": "white desk"
+ },
+ "intent": "Add a white desk to my wish list.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "white",
+ "desk"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 189
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 515,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Add a {{product}} to my wish list.",
+ "instantiation_dict": {
+ "product": "white computer desk"
+ },
+ "intent": "Add a white computer desk to my wish list.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "white",
+ "computer",
+ "desk"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 189
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 516,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/elmwood-inn-fine-teas-orange-vanilla-caffeine-free-fruit-infusion-16-ounce-pouch.html",
+ "geolocation": null,
+ "intent_template": "Add this product to my wishlist",
+ "instantiation_dict": {},
+ "intent": "Add this product to my wishlist",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Elmwood Inn Fine Teas, Orange Vanilla Caffeine-free Fruit Infusion, 16-Ounce Pouch"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 196
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 517,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/skinit-decal-gaming-skin-compatible-with-xbox-one-s-console-and-controller-bundle-officially-licensed-nfl-baltimore-ravens-design.html",
+ "geolocation": null,
+ "intent_template": "Add this product to my wishlist",
+ "instantiation_dict": {},
+ "intent": "Add this product to my wishlist",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Skinit Decal Gaming Skin Compatible with Xbox One S Console and Controller Bundle - Officially Licensed NFL Baltimore Ravens Design"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 196
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 518,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/sceptre-e195bd-srr-19-inch-720p-led-tv-true-black-2017.html",
+ "geolocation": null,
+ "intent_template": "Add this product to my wishlist",
+ "instantiation_dict": {},
+ "intent": "Add this product to my wishlist",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Sceptre E195BD-SRR 19-Inch 720P LED TV, True Black (2017)"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 196
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 519,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/iphone-13-pro-max-case-neon-turtle-iphone-13-pro-max-cases-tempered-glass-back-soft-silicone-tpu-shock-protective-case-for-apple-iphone-13-pro-max.html",
+ "geolocation": null,
+ "intent_template": "Add this product to my wishlist",
+ "instantiation_dict": {},
+ "intent": "Add this product to my wishlist",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "iPhone 13 Pro Max Case, Neon Turtle iPhone 13 Pro Max Cases, Tempered Glass Back+Soft Silicone TPU Shock Protective Case for Apple iPhone 13 Pro Max"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 196
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 520,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__/magnetic-metal-stainless-steel-d-pads-kits-directional-pad-replacement-parts-for-xbox-one-elite-controller-elite-series-2-xbox-one-xbox-one-s-x-controller.html",
+ "geolocation": null,
+ "intent_template": "Add this product to my wishlist",
+ "instantiation_dict": {},
+ "intent": "Add this product to my wishlist",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/wishlist/",
+ "locator": "document.querySelector('.products-grid.wishlist').outerText",
+ "required_contents": {
+ "must_include": [
+ "Magnetic Metal Stainless Steel D-pads Kits Directional Pad Replacement Parts for Xbox One Elite Controller, Elite Series 2, Xbox One, Xbox One S/X Controller"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 196
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 521,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Subscribe to the newsletter of OneStopMarket",
+ "instantiation_dict": {},
+ "intent": "Subscribe to the newsletter of OneStopMarket",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/newsletter/manage/",
+ "locator": "document.querySelector('[title=\"General Subscription\"').checked.toString()",
+ "required_contents": {
+ "exact_match": "true"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 199
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 522,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Fork {{repo}}.",
+ "instantiation_dict": {
+ "repo": "all repos from facebook"
+ },
+ "intent": "Fork all repos from facebook.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/projects",
+ "locator": "document.querySelector('[data-qa-selector=\"projects_list\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "create-react-app",
+ "buck"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 352
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 523,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Star the top {{number}} most stared repos in Gitlab",
+ "instantiation_dict": {
+ "number": "five"
+ },
+ "intent": "Star the top five most stared repos in Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/starred",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "AndroidSlidingUpPanel",
+ "create-react-app",
+ "ffmpeg-python",
+ "PHP_XLSXWriter",
+ "AndroidAsync"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 354
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 524,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Star the top {{number}} most stared repos in Gitlab",
+ "instantiation_dict": {
+ "number": "eight"
+ },
+ "intent": "Star the top eight most stared repos in Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/starred",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "AndroidSlidingUpPanel",
+ "create-react-app",
+ "ffmpeg-python",
+ "PHP_XLSXWriter",
+ "AndroidAsync",
+ "Pytorch-GAN",
+ "administrate",
+ "keycloak"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 354
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 525,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Star the top {{number}} most stared repos in Gitlab",
+ "instantiation_dict": {
+ "number": "four"
+ },
+ "intent": "Star the top four most stared repos in Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/starred",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "AndroidSlidingUpPanel",
+ "create-react-app",
+ "ffmpeg-python",
+ "PHP_XLSXWriter"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 354
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 526,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Star the top {{number}} most stared repos in Gitlab",
+ "instantiation_dict": {
+ "number": "three"
+ },
+ "intent": "Star the top three most stared repos in Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/starred",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "AndroidSlidingUpPanel",
+ "create-react-app",
+ "ffmpeg-python"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 354
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 527,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Star the top {{number}} most stared repos in Gitlab",
+ "instantiation_dict": {
+ "number": "one"
+ },
+ "intent": "Star the top one most stared repos in Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/starred",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "AndroidSlidingUpPanel"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 354
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 528,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "instantiation_dict": {
+ "product": "phone screen protector",
+ "time": "March 2023"
+ },
+ "intent": "Draft a refund message via their \"contact us\" form for the phone screen protector I bought March 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000180",
+ "12.99"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 154
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 529,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "instantiation_dict": {
+ "product": "bluetooth speaker",
+ "time": "Feb 2023"
+ },
+ "intent": "Draft a refund message via their \"contact us\" form for the bluetooth speaker I bought Feb 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000148",
+ "169.95"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 154
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 530,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "instantiation_dict": {
+ "product": "kitchen organizer",
+ "time": "around Feb 2023"
+ },
+ "intent": "Draft a refund message via their \"contact us\" form for the kitchen organizer I bought around Feb 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000161",
+ "68.88"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 154
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 531,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "instantiation_dict": {
+ "product": "phone case",
+ "time": "March 2023"
+ },
+ "intent": "Draft a refund message via their \"contact us\" form for the phone case I bought March 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000180",
+ "$12.99"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 154
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 532,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft a refund message via their \"contact us\" form for the {{product}} I bought {{time}}. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "instantiation_dict": {
+ "product": "PS3 remote controller",
+ "time": "early 2023"
+ },
+ "intent": "Draft a refund message via their \"contact us\" form for the PS3 remote controller I bought early 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message. Don't submit yet",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000180",
+ "1.63"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 154
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 533,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Follow {{account_list}} on Gitlab",
+ "instantiation_dict": {
+ "account_list": [
+ "convexegg",
+ "yjlou"
+ ]
+ },
+ "intent": "Follow ['convexegg', 'yjlou'] on Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/following",
+ "locator": "document.querySelector('.user-profile').outerText",
+ "required_contents": {
+ "must_include": [
+ "@convexegg",
+ "@yjlou"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 330
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 534,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Follow {{account_list}} on Gitlab",
+ "instantiation_dict": {
+ "account_list": [
+ "Jakub Klinkovsk\u00fd",
+ "Koushik",
+ "Vinta Chen"
+ ]
+ },
+ "intent": "Follow ['Jakub Klinkovsk\u00fd', 'Koushik', 'Vinta Chen'] on Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/following",
+ "locator": "document.querySelector('.user-profile').outerText",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@koush",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 330
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 535,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Follow {{account_list}} on Gitlab",
+ "instantiation_dict": {
+ "account_list": [
+ "Jakub K",
+ "ghost",
+ "Beno\u00eet Blanchon"
+ ]
+ },
+ "intent": "Follow ['Jakub K', 'ghost', 'Beno\u00eet Blanchon'] on Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/following",
+ "locator": "document.querySelector('.user-profile').outerText",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@ghost",
+ "@bblanchon"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 330
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 536,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Follow {{account_list}} on Gitlab",
+ "instantiation_dict": {
+ "account_list": [
+ "ghost",
+ "R1kk3r",
+ "Abishek"
+ ]
+ },
+ "intent": "Follow ['ghost', 'R1kk3r', 'Abishek'] on Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/following",
+ "locator": "document.querySelector('.user-profile').outerText",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@R1kk3r",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 330
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 537,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Follow {{account_list}} on Gitlab",
+ "instantiation_dict": {
+ "account_list": [
+ "Jakub Klinkovsk",
+ "convexegg",
+ "Vinta Chen",
+ "yjlou",
+ "Abishek S"
+ ]
+ },
+ "intent": "Follow ['Jakub Klinkovsk', 'convexegg', 'Vinta Chen', 'yjlou', 'Abishek S'] on Gitlab",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/users/byteblaze/following",
+ "locator": "document.querySelector('.user-profile').outerText",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@convexegg",
+ "@vinta",
+ "@yjlou",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 330
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 538,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Modify the address of order #{{order_id}} to {{address}}",
+ "instantiation_dict": {
+ "order_id": "299",
+ "address": "456 Oak Avenue, Apartment 5B, New York, NY, 10001"
+ },
+ "intent": "Modify the address of order #299 to 456 Oak Avenue, Apartment 5B, New York, NY, 10001",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/299",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "456 Oak Avenue",
+ "Apartment 5B",
+ "New York",
+ "10001"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 240
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 539,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Modify the address of order #{{order_id}} to {{address}}",
+ "instantiation_dict": {
+ "order_id": "65",
+ "address": "789 Pine Lane, San Francisco, CA, 94102"
+ },
+ "intent": "Modify the address of order #65 to 789 Pine Lane, San Francisco, CA, 94102",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/65",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "789 Pine Lane",
+ "San Francisco",
+ "California",
+ "94102"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 240
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 540,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Modify the address of order #{{order_id}} to {{address}}",
+ "instantiation_dict": {
+ "order_id": "301",
+ "address": "321 Birch Boulevard, Suite 200, Dallas, TX, 75201"
+ },
+ "intent": "Modify the address of order #301 to 321 Birch Boulevard, Suite 200, Dallas, TX, 75201",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/301",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "321 Birch Boulevard",
+ "Suite 200",
+ "Dallas",
+ "Texas",
+ "75201"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 240
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 541,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Modify the address of order #{{order_id}} to {{address}}",
+ "instantiation_dict": {
+ "order_id": "125",
+ "address": "654 Elm Drive, Apartment 12, Miami, FL, 33101"
+ },
+ "intent": "Modify the address of order #125 to 654 Elm Drive, Apartment 12, Miami, FL, 33101",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/125",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "654 Elm Drive",
+ "Apartment 12",
+ "Miami",
+ "Florida",
+ "33101"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 240
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 542,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Modify the address of order #{{order_id}} to {{address}}",
+ "instantiation_dict": {
+ "order_id": "300",
+ "address": "987 Cedar Court, Los Angeles, CA, 90012"
+ },
+ "intent": "Modify the address of order #300 to 987 Cedar Court, Los Angeles, CA, 90012",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/sales/order/view/order_id/300",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "987 Cedar Court",
+ "Los Angeles",
+ "California",
+ "90012"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 240
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 543,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update the product description of {{product}} to highlight the real user positive reviews by quoting the comments",
+ "instantiation_dict": {
+ "product": "Bella Tank"
+ },
+ "intent": "Update the product description of Bella Tank to highlight the real user positive reviews by quoting the comments",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/../bella-tank.html",
+ "locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
+ "required_contents": {
+ "must_include": [
+ "Good choice for working out and stylin' enough to wear when I'm hanging with friends on hot days",
+ "Also washes really well",
+ "Always a sweet n sporty look for the gym",
+ "Keeps me cool and the seams don't rub up against me like some of my other tanks"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 251
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 544,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
+ "instantiation_dict": {
+ "product": "Selena Yoga Hoodie"
+ },
+ "intent": "Update the description of Selena Yoga Hoodie to highlight the real user positive reviews by quoting the comments",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/../selene-yoga-hoodie.html",
+ "locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
+ "required_contents": {
+ "must_include": [
+ "I was super cold and it did the job.",
+ "The sleeves are definitely thicker than you realize, which is a good thing",
+ "really quite substantial",
+ "planning on buying another one of these in another color",
+ "the best hoodie ive ever owned"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 251
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 545,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
+ "instantiation_dict": {
+ "product": "Radiant Tee"
+ },
+ "intent": "Update the description of Radiant Tee to highlight the real user positive reviews by quoting the comments",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/../radiant-tee.html",
+ "locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
+ "required_contents": {
+ "must_include": [
+ "What I rally love here is that it does the job of keeping me cool and dry",
+ "I'm a big guy and sweat A LOT",
+ "Even after a day of gulf, I'm still dry and comfortable",
+ "What a versatile shirt",
+ "Not only does it feel very soft compared to my old worn out polos, but it also does the job promised",
+ "I like going out after my game for drinks so I look good then too and don't need to change into something fresh"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 251
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 546,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Update the description of {{product}} to highlight the real user positive reviews by quoting the comments",
+ "instantiation_dict": {
+ "product": "Lucia Cross-Fit Bra"
+ },
+ "intent": "Update the description of Lucia Cross-Fit Bra to highlight the real user positive reviews by quoting the comments",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/../affirm-water-bottle.html",
+ "locator": "document.querySelector('.data.item.content').outerText + (document.querySelector('.product.attribute.overview [itemprop=\"description\"]')?.outerText || '')",
+ "required_contents": {
+ "must_include": [
+ "Wide mouth opening makes it easy to clean"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 251
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 547,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a new {{option}} option {{value}} to the {{base_setting}} of {{product}}",
+ "instantiation_dict": {
+ "option": "color",
+ "value": "brown",
+ "base_setting": "size S",
+ "product": "Phoebe Zipper Sweatshirt"
+ },
+ "intent": "Add a new color option brown to the size S of Phoebe Zipper Sweatshirt",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1130/",
+ "locator": "document.querySelector('[data-index=\"configurable\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Phoebe Zipper Sweatshirt-S-Brown"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 252
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 548,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a new {{option}} {{value}} to {{base_setting}} of {{product}}",
+ "instantiation_dict": {
+ "option": "color",
+ "value": "blue",
+ "base_setting": "size S and M",
+ "product": "Frankie Sweatshirt"
+ },
+ "intent": "Add a new color blue to size S and M of Frankie Sweatshirt",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/110/",
+ "locator": "document.querySelector('[data-index=\"configurable\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Sweatshirt-M-Blue",
+ "Sweatshirt-S-Blue"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 252
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 549,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a new {{option}} {{value}} to {{base_setting}} {{product}}",
+ "instantiation_dict": {
+ "option": "size",
+ "value": "XXXL",
+ "base_setting": "green",
+ "product": "Minerva LumaTech V-Tee"
+ },
+ "intent": "Add a new size XXXL to green Minerva LumaTech V-Tee",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1492/",
+ "locator": "document.querySelector('[data-index=\"configurable\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "V-Tee-XXXL-Green"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 252
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 550,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a new {{option}} {{value}} to {{base_setting}} {{product}}",
+ "instantiation_dict": {
+ "option": "size",
+ "value": "XXS",
+ "base_setting": "blue and purple",
+ "product": "Nona Fitness Tank"
+ },
+ "intent": "Add a new size XXS to blue and purple Nona Fitness Tank",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1732/",
+ "locator": "document.querySelector('[data-index=\"configurable\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Tank-XXS-Blue",
+ "Tank-XXS-Purple"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 252
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 551,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add new {{option}} {{value}} to {{base_setting}} of {{product}}",
+ "instantiation_dict": {
+ "option": "size",
+ "value": "30 and 31",
+ "base_setting": "all color variants",
+ "product": "Diana Tights"
+ },
+ "intent": "Add new size 30 and 31 to all color variants of Diana Tights",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1854/",
+ "locator": "document.querySelector('[data-index=\"configurable\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Tights-30-Blue",
+ "Tights-30-Black",
+ "Tights-30-Orange",
+ "Tights-31-Blue",
+ "Tights-31-Black",
+ "Tights-31-Orange"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 252
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 552,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
+ "instantiation_dict": {
+ "directory": "real_space",
+ "subreddit": "space",
+ "gitlab_repo": "gimmiethat.space"
+ },
+ "intent": "Create a folder named real_space in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the space?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/real_space/urls.txt",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/space/134164/scientists-erupt-at-nasa-gutting-funding-for-crucial-venus",
+ "__REDDIT__/f/space/134163/virgin-orbit-fails-to-secure-funding-will-cease-operations",
+ "__REDDIT__/f/space/134162/nasa-to-name-artemis-2-crew-next-week-the-first-moon",
+ "__REDDIT__/f/space/134161/bent-light-in-deep-space-reveals-one-of-the-biggest-black",
+ "__REDDIT__/f/space/134160/seti-s-new-machine-learning-algorithm-works-like-google-s"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 84
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 553,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
+ "instantiation_dict": {
+ "directory": "news",
+ "gitlab_repo": "gimmiethat.space",
+ "subreddit": "news related subreddits"
+ },
+ "intent": "Create a folder named news in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the news related subreddits?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/news/urls.txt",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/news/129905/ohio-man-charged-for-using-molotov-cocktails-to-attack",
+ "__REDDIT__/f/news/129904/in-a-loss-for-fox-news-judge-allows-dominion-s-defamation",
+ "__REDDIT__/f/news/129903/theater-group-sues-to-block-tennessee-s-new-anti-drag-law",
+ "__REDDIT__/f/news/129902/andrew-tate-released-from-jail-in-romania-and-placed-under",
+ "__REDDIT__/f/news/129901/rare-high-risk-storm-alert-issued-for-parts-of-midwest-and"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 84
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 554,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
+ "instantiation_dict": {
+ "directory": "moive_space",
+ "gitlab_repo": "gimmiethat.space",
+ "subreddit": "movies"
+ },
+ "intent": "Create a folder named moive_space in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the movies?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/moive_space/urls.txt",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/movies/128825/scenes-in-film-that-feel-off-or-wrong-in-some-way-and-make",
+ "__REDDIT__/f/movies/128824/disney-s-live-action-lilo-amp-stitch-movie-finds-its-lilo-in",
+ "__REDDIT__/f/movies/128823/fantastic-four-movie-gets-new-writer-with-avatar-the-way-of",
+ "__REDDIT__/f/movies/128822/can-someone-explain-what-made-steven-seagal-so-appealing-for",
+ "__REDDIT__/f/movies/128821/ban-on-fetish-sex-depictions-in-film-should-end-australia"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 84
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 555,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a folder named {{directory}} in {{gitlab_repo}} repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the {{subreddit}}?",
+ "instantiation_dict": {
+ "directory": "funny_pic",
+ "gitlab_repo": "gimmiethat.space",
+ "subreddit": "memes"
+ },
+ "intent": "Create a folder named funny_pic in gimmiethat.space repo. Within it, create a file named urls.txt that contains the URLs of the 5 most recent posts from the memes?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/raw/main/funny_pic/urls.txt",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/memes/127991/it-do-be-like-that-tho",
+ "__REDDIT__/f/memes/127990/thank-you-memers-this-wouldn-t-be-possible-without-you",
+ "__REDDIT__/f/memes/127989/if-you-have-no-other-choice",
+ "__REDDIT__/f/memes/127988/yes-yes-yes",
+ "__REDDIT__/f/memes/127987/shagadelic-baby"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 84
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 556,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "nolan_honest_fans",
+ "topics": "movies directed by Christopher Nolan"
+ },
+ "intent": "Create a repo named nolan_honest_fans with movies directed by Christopher Nolan in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/nolan_honest_fans/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Following",
+ "Memento",
+ "Insomnia",
+ "Batman Begins",
+ "The Prestige",
+ "The Dark Knight",
+ "Inception",
+ "The Dark Knight Rises",
+ "Interstellar",
+ "Dunkirk",
+ "Tenet",
+ "Oppenheimer"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 557,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "nolan_old_fans",
+ "topics": "movies directed by Christopher Nolan before 2010"
+ },
+ "intent": "Create a repo named nolan_old_fans with movies directed by Christopher Nolan before 2010 in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/nolan_old_fans/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Following",
+ "Memento",
+ "Insomnia",
+ "Batman Begins",
+ "The Prestige",
+ "The Dark Knight"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 558,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "nolan_young_fans",
+ "topics": "movies directed by Christopher Nolan after 2010"
+ },
+ "intent": "Create a repo named nolan_young_fans with movies directed by Christopher Nolan after 2010 in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/nolan_young_fans/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Inception",
+ "The Dark Knight Rises",
+ "Interstellar",
+ "Dunkirk",
+ "Tenet",
+ "Oppenheimer"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 559,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "nolan_followers",
+ "topics": "career timeline of Christopher Nolan"
+ },
+ "intent": "Create a repo named nolan_followers with career timeline of Christopher Nolan in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/nolan_followers/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "1993\u20132003: Early career and breakthrough",
+ "2003\u20132013: Widespread recognition",
+ "2014\u20132019: Established Hollywood auteur",
+ "2020\u2013present"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 560,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "nolan_academy_awards",
+ "topics": "movies that won Academy Awards by Christopher Nolan"
+ },
+ "intent": "Create a repo named nolan_academy_awards with movies that won Academy Awards by Christopher Nolan in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/nolan_academy_awards/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "The Dark Knight",
+ "Inception",
+ "Interstellar",
+ "Dunkirk",
+ "Tenet"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "wikipedia"
+ ],
+ "task_id": 561,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a repo named {{name}} with {{topics}} in a README file",
+ "instantiation_dict": {
+ "name": "bafta_awards_nolan",
+ "topics": "movies that are nominated BAFTA Awards by Christopher Nolan"
+ },
+ "intent": "Create a repo named bafta_awards_nolan with movies that are nominated BAFTA Awards by Christopher Nolan in a README file",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/bafta_awards_nolan/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Batman Begins",
+ "The Dark Knight",
+ "Inception",
+ "The Dark Knight Rises",
+ "Interstellar",
+ "Dunkirk",
+ "Tenet"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 87
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 562,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
+ "instantiation_dict": {
+ "name": "Awesome_DIY_ideas",
+ "num": 6
+ },
+ "intent": "create a repository named Awesome_DIY_ideas that includes a README file with the links to the most active 6 DIY ideas on DIY subreddit?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/Awesome_DIY_ideas/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
+ "__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
+ "__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
+ "__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
+ "__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
+ "__REDDIT__/f/DIY/118931/afci-outlet-question"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 88
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 563,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
+ "instantiation_dict": {
+ "name": "fun_thing_to_do",
+ "num": 5
+ },
+ "intent": "create a repository named fun_thing_to_do that includes a README file with the links to the most active 5 DIY ideas on DIY subreddit?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/fun_thing_to_do/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
+ "__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
+ "__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
+ "__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
+ "__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 88
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 564,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
+ "instantiation_dict": {
+ "name": "live_a_life",
+ "num": 3
+ },
+ "intent": "create a repository named live_a_life that includes a README file with the links to the most active 3 DIY ideas on DIY subreddit?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/live_a_life/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
+ "__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
+ "__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 88
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 565,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
+ "instantiation_dict": {
+ "name": "TODO",
+ "num": 10
+ },
+ "intent": "create a repository named TODO that includes a README file with the links to the most active 10 DIY ideas on DIY subreddit?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/TODO/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
+ "__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
+ "__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
+ "__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
+ "__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
+ "__REDDIT__/f/DIY/118931/afci-outlet-question",
+ "__REDDIT__/f/DIY/118824/teflon-tape-to-attach-washing-machine-drain-hose-to-pipe",
+ "__REDDIT__/f/DIY/118866/paver-base-for-shed",
+ "__REDDIT__/f/DIY/118820/ways-to-locate-our-buried-electrical-service",
+ "__REDDIT__/f/DIY/118836/how-to-eliminate-transitions-for-disability-mobility-reasons"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 88
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 566,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a repository named {{name}} that includes a README file with the links to the most active {{num}} DIY ideas on DIY subreddit?",
+ "instantiation_dict": {
+ "name": "Do it myself",
+ "num": 8
+ },
+ "intent": "create a repository named Do it myself that includes a README file with the links to the most active 8 DIY ideas on DIY subreddit?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/Do-it-myself/-/raw/main/README.md",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "__REDDIT__/f/DIY/118903/separate-glued-plastic-parts",
+ "__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess",
+ "__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing",
+ "__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit",
+ "__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches",
+ "__REDDIT__/f/DIY/118931/afci-outlet-question",
+ "__REDDIT__/f/DIY/118824/teflon-tape-to-attach-washing-machine-drain-hose-to-pipe",
+ "__REDDIT__/f/DIY/118866/paver-base-for-shed"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 88
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 567,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
+ "instantiation_dict": {
+ "collaborator_account_list": "Jakub Klinkovsk\u00fd and Beno\u00eet Blanchon",
+ "repo": "gimmiethat.space"
+ },
+ "intent": "Invite Jakub Klinkovsk\u00fd and Beno\u00eet Blanchon as collaborator to gimmiethat.space repo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@bblanchon"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 293
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 568,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
+ "instantiation_dict": {
+ "collaborator_account_list": "Abishek and Vinta",
+ "repo": "a11yproject.com"
+ },
+ "intent": "Invite Abishek and Vinta as collaborator to a11yproject.com repo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/a11yproject/a11yproject.com/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@abisubramanya27",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 293
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 569,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
+ "instantiation_dict": {
+ "collaborator_account_list": "Beno\u00eet and Abishek",
+ "repo": "my HTML5 markup extention"
+ },
+ "intent": "Invite Beno\u00eet and Abishek as collaborator to my HTML5 markup extention repo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/accessible-html-content-patterns/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@bblanchon",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 293
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 570,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Invite {{collaborator_account_list}} as collaborator to {{repo}} repo",
+ "instantiation_dict": {
+ "collaborator_account_list": "Jakub K, Alex Dills, Alex Hutnik and Beno\u00eet Blanchon",
+ "repo": "my time tracking tool project"
+ },
+ "intent": "Invite Jakub K, Alex Dills, Alex Hutnik and Beno\u00eet Blanchon as collaborator to my time tracking tool project repo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/timeit/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@lahwaacz",
+ "@V13Axel",
+ "@alexhutnik",
+ "@bblanchon"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 293
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 571,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
+ "instantiation_dict": {
+ "address": "231 Willow Way, Suite 100, Chicago, IL, 60601"
+ },
+ "intent": "I recently moved, my address is 231 Willow Way, Suite 100, Chicago, IL, 60601, update my information on OneStopShopping accordingly",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "231 Willow Way",
+ "Suite 100",
+ "Chicago, Illinois, 60601"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "231 Willow Way",
+ "Suite 100",
+ "Chicago, Illinois, 60601"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 165
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 572,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
+ "instantiation_dict": {
+ "address": "654 Aspen Road, House #3, Boston, MA, 02110"
+ },
+ "intent": "I recently moved, my address is 654 Aspen Road, House #3, Boston, MA, 02110, update my information on OneStopShopping accordingly",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "654 Aspen Road",
+ "House #3",
+ "Boston, Massachusetts, 02110"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "654 Aspen Road",
+ "House #3",
+ "Boston, Massachusetts, 02110"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 165
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 573,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
+ "instantiation_dict": {
+ "address": "987 Sycamore Circle, Philadelphia, PA, 19102"
+ },
+ "intent": "I recently moved, my address is 987 Sycamore Circle, Philadelphia, PA, 19102, update my information on OneStopShopping accordingly",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "987 Sycamore Circle",
+ "Philadelphia, Pennsylvania, 19102"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "987 Sycamore Circle",
+ "Philadelphia, Pennsylvania, 19102"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 165
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 574,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
+ "instantiation_dict": {
+ "address": "111 Magnolia Path, Atlanta, GA, 30303"
+ },
+ "intent": "I recently moved, my address is 111 Magnolia Path, Atlanta, GA, 30303, update my information on OneStopShopping accordingly",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "111 Magnolia Path",
+ "Atlanta, Georgia, 30303"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "111 Magnolia Path",
+ "Atlanta, Georgia, 30303"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 165
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 575,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "I recently moved, my address is {{address}}, update my information on OneStopShopping accordingly",
+ "instantiation_dict": {
+ "address": "222 Redwood Rise, Suite 300, Seattle, WA, 98101"
+ },
+ "intent": "I recently moved, my address is 222 Redwood Rise, Suite 300, Seattle, WA, 98101, update my information on OneStopShopping accordingly",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-shipping > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "222 Redwood Rise",
+ "Suite 300",
+ "Seattle, Washington, 98101"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING__/customer/address",
+ "locator": "document.querySelector(\".box.box-address-billing > .box-content\").outerText",
+ "required_contents": {
+ "must_include": [
+ "222 Redwood Rise",
+ "Suite 300",
+ "Seattle, Washington, 98101"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 165
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 576,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
+ "instantiation_dict": {
+ "repo": "a11y-webring.club",
+ "role": "developer",
+ "user_list": [
+ "abisubramanya27",
+ "lahwaacz"
+ ]
+ },
+ "intent": "Add the following users to repo a11y-webring.club as developer: ['abisubramanya27', 'lahwaacz']",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/a11y-webring.club/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
+ "required_contents": {
+ "must_include": [
+ "Developer"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/a11y-webring.club/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
+ "required_contents": {
+ "must_include": [
+ "Developer"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 351
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 577,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Add the following users to my {{repo}} as {{role}}: {{user_list}}",
+ "instantiation_dict": {
+ "repo": "GitHub timeline item management extension",
+ "role": "maintainer",
+ "user_list": [
+ "abisubramanya27",
+ "lahwaacz"
+ ]
+ },
+ "intent": "Add the following users to my GitHub timeline item management extension as maintainer: ['abisubramanya27', 'lahwaacz']",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/remove-board-movement-events-from-the-github-issue-timeline/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'abisubramanya27')",
+ "required_contents": {
+ "must_include": [
+ "Maintainer"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/remove-board-movement-events-from-the-github-issue-timeline/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'lahwaacz')",
+ "required_contents": {
+ "must_include": [
+ "Maintainer"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 351
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 578,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
+ "instantiation_dict": {
+ "repo": "millennials-to-snake-people",
+ "role": "reporter",
+ "user_list": [
+ "yjlou",
+ "a11yproject"
+ ]
+ },
+ "intent": "Add the following users to repo millennials-to-snake-people as reporter: ['yjlou', 'a11yproject']",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/millennials-to-snake-people/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
+ "required_contents": {
+ "must_include": [
+ "Reporter"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/millennials-to-snake-people/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'a11yproject')",
+ "required_contents": {
+ "must_include": [
+ "Reporter"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 351
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 579,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Add the following users to my {{repo}} as {{role}}: {{user_list}}",
+ "instantiation_dict": {
+ "repo": "time tracking tool",
+ "role": "guest",
+ "user_list": [
+ "yjlou"
+ ]
+ },
+ "intent": "Add the following users to my time tracking tool as guest: ['yjlou']",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/timeit/-/project_members",
+ "locator": "func:gitlab_get_project_memeber_role(__page__, 'yjlou')",
+ "required_contents": {
+ "must_include": [
+ "Guest"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 351
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 580,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
+ "instantiation_dict": {
+ "name": "sci_fi",
+ "description": "A wild place for sci-fi enthusiasts",
+ "sidebar_list": [
+ "New",
+ "Classic",
+ "Movies",
+ "Post my novel",
+ "Random"
+ ]
+ },
+ "intent": "Create a new forum named sci_fi, with a description of A wild place for sci-fi enthusiasts, and include ['New', 'Classic', 'Movies', 'Post my novel', 'Random'] in the sidebar?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/sci_fi/edit",
+ "locator": "document.querySelector(\"#forum_description\").value",
+ "required_contents": {
+ "must_include": [
+ "A wild place for sci-fi enthusiasts"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/sci_fi/edit",
+ "locator": "document.querySelector(\"#forum_sidebar\").value",
+ "required_contents": {
+ "must_include": [
+ "New",
+ "Classic",
+ "Movies",
+ "Post my novel",
+ "Random"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 7
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 581,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
+ "instantiation_dict": {
+ "name": "cmu_lti",
+ "description": "Language Technologies Institute at Carnegie Mellon University",
+ "sidebar_list": [
+ "announcement",
+ "paper",
+ "alumni"
+ ]
+ },
+ "intent": "Create a new forum named cmu_lti, with a description of Language Technologies Institute at Carnegie Mellon University, and include ['announcement', 'paper', 'alumni'] in the sidebar?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/cmu_lti/edit",
+ "locator": "document.querySelector(\"#forum_description\").value",
+ "required_contents": {
+ "must_include": [
+ "Language Technologies Institute at Carnegie Mellon University"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/cmu_lti/edit",
+ "locator": "document.querySelector(\"#forum_sidebar\").value",
+ "required_contents": {
+ "must_include": [
+ "announcement",
+ "paper",
+ "alumni"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 7
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 582,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
+ "instantiation_dict": {
+ "name": "Cyberpunk",
+ "description": "Welcome to the future",
+ "sidebar_list": [
+ "Games",
+ "Books",
+ "Movies",
+ "Future"
+ ]
+ },
+ "intent": "Create a new forum named Cyberpunk, with a description of Welcome to the future, and include ['Games', 'Books', 'Movies', 'Future'] in the sidebar?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/Cyberpunk/edit",
+ "locator": "document.querySelector(\"#forum_description\").value",
+ "required_contents": {
+ "must_include": [
+ "Welcome to the future"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/Cyberpunk/edit",
+ "locator": "document.querySelector(\"#forum_sidebar\").value",
+ "required_contents": {
+ "must_include": [
+ "Games",
+ "Books",
+ "Movies",
+ "Future"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 7
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 583,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
+ "instantiation_dict": {
+ "name": "PlantsForCatParents",
+ "description": "Cat parents & plant lovers",
+ "sidebar_list": [
+ "Cat friendly",
+ "Local vendors",
+ "Promotion",
+ "Toxic plants!"
+ ]
+ },
+ "intent": "Create a new forum named PlantsForCatParents, with a description of Cat parents & plant lovers, and include ['Cat friendly', 'Local vendors', 'Promotion', 'Toxic plants!'] in the sidebar?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/PlantsForCatParents/edit",
+ "locator": "document.querySelector(\"#forum_description\").value",
+ "required_contents": {
+ "must_include": [
+ "Cat parents & plant lovers"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/PlantsForCatParents/edit",
+ "locator": "document.querySelector(\"#forum_sidebar\").value",
+ "required_contents": {
+ "must_include": [
+ "Cat friendly",
+ "Local vendors",
+ "Promotion",
+ "Toxic plants!"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 7
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 584,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a new forum named {{name}}, with a description of {{description}}, and include {{sidebar_list}} in the sidebar?",
+ "instantiation_dict": {
+ "name": "Karaoke",
+ "description": "Place for Karaoke lovers",
+ "sidebar_list": [
+ "devices",
+ "setup"
+ ]
+ },
+ "intent": "Create a new forum named Karaoke, with a description of Place for Karaoke lovers, and include ['devices', 'setup'] in the sidebar?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/Karaoke/edit",
+ "locator": "document.querySelector(\"#forum_description\").value",
+ "required_contents": {
+ "must_include": [
+ "Place for Karaoke lovers"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/Karaoke/edit",
+ "locator": "document.querySelector(\"#forum_sidebar\").value",
+ "required_contents": {
+ "must_include": [
+ "devices",
+ "setup"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 7
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 585,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
+ "instantiation_dict": {
+ "product": "floor lamp",
+ "num_star": 5,
+ "nickname": "Emma Lopez"
+ },
+ "intent": "Rate my recent purchase of floor lamp with 5 stars, using my nickname Emma Lopez?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_rating('B00J8RZL7I')",
+ "required_contents": {
+ "must_include": [
+ "100"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_author('B00J8RZL7I')",
+ "required_contents": {
+ "must_include": [
+ "Emma Lopez"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 194
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 586,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
+ "instantiation_dict": {
+ "product": "Jiffy Corn Muffin Cornbread Mix",
+ "num_star": 4,
+ "nickname": "ShoppingEmma"
+ },
+ "intent": "Rate my recent purchase of Jiffy Corn Muffin Cornbread Mix with 4 stars, using my nickname ShoppingEmma?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_rating('B07HZB38XH')",
+ "required_contents": {
+ "must_include": [
+ "80"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_author('B07HZB38XH')",
+ "required_contents": {
+ "must_include": [
+ "ShoppingEmma"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 194
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 587,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
+ "instantiation_dict": {
+ "product": "PS3 Remote Controllers",
+ "num_star": 3,
+ "nickname": "GamingEmma"
+ },
+ "intent": "Rate my recent purchase of PS3 Remote Controllers with 3 stars, using my nickname GamingEmma?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_rating('B0041MSF2S')",
+ "required_contents": {
+ "must_include": [
+ "60"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_author('B0041MSF2S')",
+ "required_contents": {
+ "must_include": [
+ "GamingEmma"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 194
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 588,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
+ "instantiation_dict": {
+ "product": "Foundation For Mattress With Frame Set",
+ "num_star": 1,
+ "nickname": "ShoppingEmma"
+ },
+ "intent": "Rate my recent purchase of Foundation For Mattress With Frame Set with 1 stars, using my nickname ShoppingEmma?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_rating('B07DFJ5XKH')",
+ "required_contents": {
+ "must_include": [
+ "20"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_author('B07DFJ5XKH')",
+ "required_contents": {
+ "must_include": [
+ "ShoppingEmma"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 194
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 589,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Rate my recent purchase of {{product}} with {{num_star}} stars, using my nickname {{nickname}}?",
+ "instantiation_dict": {
+ "product": "Mini Wireless Bluetooth Speaker",
+ "num_star": 2,
+ "nickname": "SimpleEmma"
+ },
+ "intent": "Rate my recent purchase of Mini Wireless Bluetooth Speaker with 2 stars, using my nickname SimpleEmma?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_rating('B09P7BFL4H')",
+ "required_contents": {
+ "must_include": [
+ "40"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "func:shopping_get_sku_latest_review_author('B09P7BFL4H')",
+ "required_contents": {
+ "must_include": [
+ "SimpleEmma"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 194
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 590,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
+ "instantiation_dict": {
+ "event": "event of product launch",
+ "start_date": "1/16/2023",
+ "end_date": "1/30/2023"
+ },
+ "intent": "Create a milestone for the upcoming event of product launch starting on 1/16/2023 and ending on 1/30/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/milestones",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"#content-body\").outerText",
+ "required_contents": {
+ "must_include": [
+ "product launch"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.start_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Jan 16, 2030"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.due_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Jan 30, 2030"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 339
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 591,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
+ "instantiation_dict": {
+ "event": "practice of collective code review",
+ "start_date": "1/16/2023",
+ "end_date": "in 20 days"
+ },
+ "intent": "Create a milestone for the upcoming practice of collective code review starting on 1/16/2023 and ending on in 20 days",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/milestones",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"#content-body\").outerText",
+ "required_contents": {
+ "must_include": [
+ "code review"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.start_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Jan 16, 2030"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.due_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Feb 5, 2030"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 339
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 592,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
+ "instantiation_dict": {
+ "event": "task of cleaning sensitive information",
+ "start_date": "2/16/2023",
+ "end_date": "in 20 days"
+ },
+ "intent": "Create a milestone for the upcoming task of cleaning sensitive information starting on 2/16/2023 and ending on in 20 days",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/milestones",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"#content-body\").outerText",
+ "required_contents": {
+ "must_include": [
+ "sensitive information"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.start_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Feb 16, 2030"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.due_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Mar 8, 2030"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 339
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 593,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/dotfiles",
+ "geolocation": null,
+ "intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
+ "instantiation_dict": {
+ "event": "task of merging all branches to main",
+ "start_date": "March 15, 2044",
+ "end_date": "March 30, 2044"
+ },
+ "intent": "Create a milestone for the upcoming task of merging all branches to main starting on March 15, 2044 and ending on March 30, 2044",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/dotfiles/-/milestones",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"#content-body\").outerText",
+ "required_contents": {
+ "must_include": [
+ "all branches to main"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.start_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Mar 15, 2044"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.due_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "Mar 30, 2044"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 339
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 594,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/dotfiles",
+ "geolocation": null,
+ "intent_template": "Create a milestone for the upcoming {{event}} starting on {{start_date}} and ending on {{end_date}}",
+ "instantiation_dict": {
+ "event": "task of adding a new branch for zsh comprehensive support",
+ "start_date": "5/1/2044",
+ "end_date": "in 20 days"
+ },
+ "intent": "Create a milestone for the upcoming task of adding a new branch for zsh comprehensive support starting on 5/1/2044 and ending on in 20 days",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/dotfiles/-/milestones",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"#content-body\").outerText",
+ "required_contents": {
+ "must_include": [
+ "zsh comprehensive support"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.start_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "May 1, 2044"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.due_date').outerText",
+ "required_contents": {
+ "must_include": [
+ "May 21, 2044"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 339
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 595,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
+ "instantiation_dict": {
+ "subreddit": "space"
+ },
+ "intent": "Open the thread of a trending post on the forum \"space\" and subscribe.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/space",
+ "program_html": [
+ {
+ "url": "__REDDIT__/",
+ "locator": "document.querySelector(\"#sidebar > section\").outerText",
+ "required_contents": {
+ "must_include": [
+ "space"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 4
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 596,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
+ "instantiation_dict": {
+ "subreddit": "books"
+ },
+ "intent": "Open the thread of a trending post on the forum \"books\" and subscribe.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "__REDDIT__/",
+ "locator": "document.querySelector(\"#sidebar > section\").outerText",
+ "required_contents": {
+ "must_include": [
+ "books"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 4
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 597,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
+ "instantiation_dict": {
+ "subreddit": "consoles"
+ },
+ "intent": "Open the thread of a trending post on the forum \"consoles\" and subscribe.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/consoles",
+ "program_html": [
+ {
+ "url": "__REDDIT__/",
+ "locator": "document.querySelector(\"#sidebar > section\").outerText",
+ "required_contents": {
+ "must_include": [
+ "consoles"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 4
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 598,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
+ "instantiation_dict": {
+ "subreddit": "pittsburgh"
+ },
+ "intent": "Open the thread of a trending post on the forum \"pittsburgh\" and subscribe.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/pittsburgh",
+ "program_html": [
+ {
+ "url": "__REDDIT__/",
+ "locator": "document.querySelector(\"#sidebar > section\").outerText",
+ "required_contents": {
+ "must_include": [
+ "pittsburgh"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 4
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 599,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Open the thread of a trending post on the forum \"{{subreddit}}\" and subscribe.",
+ "instantiation_dict": {
+ "subreddit": "machine learning"
+ },
+ "intent": "Open the thread of a trending post on the forum \"machine learning\" and subscribe.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/MachineLearning",
+ "program_html": [
+ {
+ "url": "__REDDIT__/",
+ "locator": "document.querySelector(\"#sidebar > section\").outerText",
+ "required_contents": {
+ "must_include": [
+ "MachineLearning"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 4
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 600,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
+ "instantiation_dict": {
+ "question": "what is the recommended console to buy these days"
+ },
+ "intent": "Post my question, \"what is the recommended console to buy these days\", in a subreddit where I'm likely to get an answer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/consoles",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "what is the recommended console to buy these days"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 3765
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 601,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
+ "instantiation_dict": {
+ "question": "is car necessary in NYC"
+ },
+ "intent": "Post my question, \"is car necessary in NYC\", in a subreddit where I'm likely to get an answer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/nyc",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "is car necessary in NYC |OR| is a car necessary in NYC"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 3765
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 602,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
+ "instantiation_dict": {
+ "question": "places for new drivers to learn driving in pittsburgh"
+ },
+ "intent": "Post my question, \"places for new drivers to learn driving in pittsburgh\", in a subreddit where I'm likely to get an answer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/pittsburgh",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "places for new drivers to learn driving in pittsburgh"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 3765
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 603,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
+ "instantiation_dict": {
+ "question": "safe and budget apartment to live in nyc"
+ },
+ "intent": "Post my question, \"safe and budget apartment to live in nyc\", in a subreddit where I'm likely to get an answer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/nyc",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "safe and budget apartment to live in nyc"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 3765
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 604,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post my question, \"{{question}}\", in a subreddit where I'm likely to get an answer",
+ "instantiation_dict": {
+ "question": "what is the SOTA web navigation agent repo"
+ },
+ "intent": "Post my question, \"what is the SOTA web navigation agent repo\", in a subreddit where I'm likely to get an answer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/deeplearning |OR| __REDDIT__/f/MachineLearning |OR| __REDDIT__/f/singularity",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "what is the SOTA web navigation agent repo"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 3765
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 605,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
+ "instantiation_dict": {
+ "topic": "gaming consoles",
+ "question": "what is the recommended console to buy these days"
+ },
+ "intent": "Find a subreddit focused on topics related to gaming consoles, and post my question, \"what is the recommended console to buy these days\" there",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/consoles",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "what is the recommended console to buy these days"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 5
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 606,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
+ "instantiation_dict": {
+ "topic": "NYC",
+ "question": "is car necessary"
+ },
+ "intent": "Find a subreddit focused on topics related to NYC, and post my question, \"is car necessary\" there",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/nyc",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "is car necessary |OR| is a car necessary"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 5
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 607,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
+ "instantiation_dict": {
+ "topic": "city Pittsburgh",
+ "question": "places for new drivers to learn driving"
+ },
+ "intent": "Find a subreddit focused on topics related to city Pittsburgh, and post my question, \"places for new drivers to learn driving\" there",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/pittsburgh",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "places for new drivers to learn driving"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 5
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 608,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
+ "instantiation_dict": {
+ "topic": "city lives in DMV area",
+ "question": "safe and budget apartment to live"
+ },
+ "intent": "Find a subreddit focused on topics related to city lives in DMV area, and post my question, \"safe and budget apartment to live\" there",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/washington |OR| __REDDIT__/f/washingtondc",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "safe and budget apartment to live"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 5
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 609,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a subreddit focused on topics related to {{topic}}, and post my question, \"{{question}}\" there",
+ "instantiation_dict": {
+ "topic": "ML, DL, NLP",
+ "question": "what is the SOTA web navigation agent repo"
+ },
+ "intent": "Find a subreddit focused on topics related to ML, DL, NLP, and post my question, \"what is the SOTA web navigation agent repo\" there",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/deeplearning |OR| __REDDIT__/f/MachineLearning |OR| __REDDIT__/f/singularity",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "what is the SOTA web navigation agent repo"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 5
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 610,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
+ "instantiation_dict": {
+ "book": "To Kill a Mockingbird by Harper Lee",
+ "content": "good book!"
+ },
+ "intent": "Post a review of my recent reading \"To Kill a Mockingbird by Harper Lee\" in the r/books with my comment \"good book!\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "To Kill a Mockingbird by Harper Lee",
+ "good book!"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 9
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 611,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
+ "instantiation_dict": {
+ "book": "Harry Potter",
+ "content": "Wonderful journey"
+ },
+ "intent": "Post a review of my recent reading \"Harry Potter\" in the r/books with my comment \"Wonderful journey\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Harry Potter",
+ "Wonderful journey"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 9
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 612,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
+ "instantiation_dict": {
+ "book": "big little lies",
+ "content": "can't stop it"
+ },
+ "intent": "Post a review of my recent reading \"big little lies\" in the r/books with my comment \"can't stop it\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "big little lies",
+ "can't stop it"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 9
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 613,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
+ "instantiation_dict": {
+ "book": "Love story",
+ "content": "I cried"
+ },
+ "intent": "Post a review of my recent reading \"Love story\" in the r/books with my comment \"I cried\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Love story",
+ "I cried"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 9
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 614,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a review of my recent reading \"{{book}}\" in the r/books with my comment \"{{content}}\".",
+ "instantiation_dict": {
+ "book": "Gone with the wind",
+ "content": "It's a book with history"
+ },
+ "intent": "Post a review of my recent reading \"Gone with the wind\" in the r/books with my comment \"It's a book with history\".",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Gone with the wind",
+ "It's a book with history"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 9
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 615,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pics",
+ "geolocation": null,
+ "intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
+ "instantiation_dict": {
+ "content": "Bald Eagle",
+ "subreddit": "earthporn"
+ },
+ "intent": "Re-post the image of Bald Eagle in this page to earthporn subreddit and note \"from /f/pics\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/earthporn",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "from /f/pics"
+ ]
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "b02113033af32feae9ff147dbbe3764039368d67d193885bd04e65c2e6beea9c.jpg"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 11
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 616,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pics",
+ "geolocation": null,
+ "intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
+ "instantiation_dict": {
+ "content": "Thanksgiving turkey",
+ "subreddit": "funny"
+ },
+ "intent": "Re-post the image of Thanksgiving turkey in this page to funny subreddit and note \"from /f/pics\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/funny",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "from /f/pics"
+ ]
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "2e4fa0a328e653a97a7d07046291c298ef5b4e0d0c73a287f317ca86a8e8685f.jpg"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 11
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 617,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pics",
+ "geolocation": null,
+ "intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
+ "instantiation_dict": {
+ "content": "Firework",
+ "subreddit": "earthporn"
+ },
+ "intent": "Re-post the image of Firework in this page to earthporn subreddit and note \"from /f/pics\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/earthporn",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "from /f/pics"
+ ]
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "92411be6af4e9ad5ccd3ccbaa01c10457bb00e704e99c58dd430de1a958307fd.jpg"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 11
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 618,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pics",
+ "geolocation": null,
+ "intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
+ "instantiation_dict": {
+ "content": "Wife's costume",
+ "subreddit": "funny"
+ },
+ "intent": "Re-post the image of Wife's costume in this page to funny subreddit and note \"from /f/pics\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/funny",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "from /f/pics"
+ ]
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "6bfbf1280d28d84a9261695f0cac5a90addaaff1174807a2b381fdc159f3ed00.jpg"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 11
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 619,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pics",
+ "geolocation": null,
+ "intent_template": "Re-post the image of {{content}} in this page to {{subreddit}} subreddit and note \"from /f/pics\"",
+ "instantiation_dict": {
+ "content": "costume contest",
+ "subreddit": "funny"
+ },
+ "intent": "Re-post the image of costume contest in this page to funny subreddit and note \"from /f/pics\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/funny",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "from /f/pics"
+ ]
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "[...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "bd8bc5f4c846aac4df08626faa3a34a7d47c8f3bdd92bf615a54afd939f063a7.jpg"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 11
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 620,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
+ "instantiation_dict": {
+ "issue": "deal with long-distance relationships"
+ },
+ "intent": "Ask for advice about deal with long-distance relationships in a subreddit for relations",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "long",
+ "distance",
+ "relation"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 12
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 621,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
+ "instantiation_dict": {
+ "issue": "cheat"
+ },
+ "intent": "Ask for advice about cheat in a subreddit for relations",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "cheat"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 12
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 622,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
+ "instantiation_dict": {
+ "issue": "sexual harassment"
+ },
+ "intent": "Ask for advice about sexual harassment in a subreddit for relations",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "sexual",
+ "harassment"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 12
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 623,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
+ "instantiation_dict": {
+ "issue": "gift for birthday"
+ },
+ "intent": "Ask for advice about gift for birthday in a subreddit for relations",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "gift",
+ "birthday"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 12
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 624,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for advice about {{issue}} in a subreddit for relations",
+ "instantiation_dict": {
+ "issue": "break-up remedy"
+ },
+ "intent": "Ask for advice about break-up remedy in a subreddit for relations",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "break",
+ "remedy"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 12
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 625,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "instantiation_dict": {
+ "topic": "the effectiveness of online learning"
+ },
+ "intent": "Create a discussion post about \"the effectiveness of online learning\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "your opinion",
+ "the effectiveness of online learning"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 13
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 626,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "instantiation_dict": {
+ "topic": "Iphone 14"
+ },
+ "intent": "Create a discussion post about \"Iphone 14\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/iphone",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "your opinion",
+ "Iphone 14"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 13
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 627,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "instantiation_dict": {
+ "topic": "Harry Potter movie series"
+ },
+ "intent": "Create a discussion post about \"Harry Potter movie series\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/movies",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "your opinion",
+ "Harry Potter movie series"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 13
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 628,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "instantiation_dict": {
+ "topic": "long distance relationship"
+ },
+ "intent": "Create a discussion post about \"long distance relationship\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/relationship_advice",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "your opinion",
+ "long distance relationship"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 13
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 629,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Create a discussion post about \"{{topic}}\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "instantiation_dict": {
+ "topic": "Fun thing to do in Pittsburgh"
+ },
+ "intent": "Create a discussion post about \"Fun thing to do in Pittsburgh\" in a relevant subreddit and ask users for their opinions with the simple prompt, \"your opinion\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/pittsburgh",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "your opinion",
+ "Fun thing to do in Pittsburgh |OR| Fun things to do in Pittsburgh"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 13
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 630,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
+ "instantiation_dict": {
+ "category": "noise-cancelling headphones",
+ "price": "$200",
+ "subreddit": "r/headphones"
+ },
+ "intent": "Ask for product recommendations for noise-cancelling headphones within a budget of $200 in r/headphones",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/headphones",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "200",
+ "noise-cancelling",
+ "headphone"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 15
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 631,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
+ "instantiation_dict": {
+ "category": "running shoes",
+ "price": "$100",
+ "subreddit": "r/sports"
+ },
+ "intent": "Ask for product recommendations for running shoes within a budget of $100 in r/sports",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/sports",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "100",
+ "running",
+ "shoes"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 15
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 632,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
+ "instantiation_dict": {
+ "category": "running shoes",
+ "price": "$500",
+ "subreddit": "r/sports"
+ },
+ "intent": "Ask for product recommendations for running shoes within a budget of $500 in r/sports",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/sports",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "500",
+ "running",
+ "shoes"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 15
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 633,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
+ "instantiation_dict": {
+ "category": "running pants",
+ "price": "$500",
+ "subreddit": "r/sports"
+ },
+ "intent": "Ask for product recommendations for running pants within a budget of $500 in r/sports",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/sports",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "500",
+ "running",
+ "pants"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 15
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 634,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Ask for product recommendations for {{category}} within a budget of {{price}} in {{subreddit}}",
+ "instantiation_dict": {
+ "category": "used iphone",
+ "price": "$1000",
+ "subreddit": "r/iphone"
+ },
+ "intent": "Ask for product recommendations for used iphone within a budget of $1000 in r/iphone",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/iphone",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "1000",
+ "used iphone"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 15
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 635,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
+ "instantiation_dict": {
+ "category": "noise-cancelling headphones",
+ "price": "$200"
+ },
+ "intent": "Post in the most appropriate subreddit and ask for recommendations for noise-cancelling headphones products within a budget of $200",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/headphones",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "200",
+ "noise-cancelling",
+ "headphone"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 6100
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 636,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
+ "instantiation_dict": {
+ "category": "DIY toolkit",
+ "price": "$100"
+ },
+ "intent": "Post in the most appropriate subreddit and ask for recommendations for DIY toolkit products within a budget of $100",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/DIY",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "100",
+ "DIY",
+ "toolkit"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 6100
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 637,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
+ "instantiation_dict": {
+ "category": "sony headphones",
+ "price": "$500"
+ },
+ "intent": "Post in the most appropriate subreddit and ask for recommendations for sony headphones products within a budget of $500",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/headphones",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "500",
+ "sony headphone"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 6100
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 638,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
+ "instantiation_dict": {
+ "category": "must-have product in my life",
+ "price": "$30"
+ },
+ "intent": "Post in the most appropriate subreddit and ask for recommendations for must-have product in my life products within a budget of $30",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/BuyItForLife",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "30",
+ "must-have",
+ "product",
+ "life"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 6100
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 639,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in the most appropriate subreddit and ask for recommendations for {{category}} products within a budget of {{price}}",
+ "instantiation_dict": {
+ "category": "used iphone",
+ "price": "$1000"
+ },
+ "intent": "Post in the most appropriate subreddit and ask for recommendations for used iphone products within a budget of $1000",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/iphone",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "1000",
+ "used iphone"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 6100
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 640,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "interest": "book reading",
+ "date": "March 15th",
+ "subreddit": "r/books"
+ },
+ "intent": "Post a notice on a virtual meetup for book reading enthusiasts on March 15th in the r/books subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "book reading",
+ "Mar 15th |OR| March 15th",
+ "virtual",
+ "meetup"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 16
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 641,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "interest": "Harry Potter",
+ "date": "July 8th",
+ "subreddit": "books"
+ },
+ "intent": "Post a notice on a virtual meetup for Harry Potter enthusiasts on July 8th in the books subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Harry Potter",
+ "Jul 8th |OR| July 8th",
+ "virtual",
+ "meetup"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 16
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 642,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "interest": "Big little lies",
+ "date": "Sep 10th",
+ "subreddit": "books"
+ },
+ "intent": "Post a notice on a virtual meetup for Big little lies enthusiasts on Sep 10th in the books subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Big little lies",
+ "Sep 10th |OR| September 10th",
+ "virtual",
+ "meetup"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 16
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 643,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "interest": "racing cars",
+ "date": "Oct 21st",
+ "subreddit": "nyc"
+ },
+ "intent": "Post a notice on a virtual meetup for racing cars enthusiasts on Oct 21st in the nyc subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/nyc",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "racing cars",
+ "Oct 21st |OR| October 21st",
+ "virtual",
+ "meetup"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 16
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 644,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post a notice on a virtual meetup for {{interest}} enthusiasts on {{date}} in the {{subreddit}} subreddit",
+ "instantiation_dict": {
+ "interest": "Tears of Kingdom",
+ "date": "Dec 15th",
+ "subreddit": "gaming"
+ },
+ "intent": "Post a notice on a virtual meetup for Tears of Kingdom enthusiasts on Dec 15th in the gaming subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "Tears of Kingdom",
+ "Dec 15th |OR| December 15th",
+ "virtual",
+ "meetup"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 16
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 645,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in {{subreddit}} subreddit about what could machine learning help the corresponding field.",
+ "instantiation_dict": {
+ "subreddit": "books"
+ },
+ "intent": "Post in books subreddit about what could machine learning help the corresponding field.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "machine learning",
+ "help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 19
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 646,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in {{subreddit}} subreddit about what could midjourney help the corresponding field.",
+ "instantiation_dict": {
+ "subreddit": "DIY"
+ },
+ "intent": "Post in DIY subreddit about what could midjourney help the corresponding field.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/diy",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "midjourney",
+ "help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 19
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 647,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in {{subreddit}} forum about what could open-source LLMs help the corresponding field.",
+ "instantiation_dict": {
+ "subreddit": "technology"
+ },
+ "intent": "Post in technology forum about what could open-source LLMs help the corresponding field.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/technology",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "open-source LLMs",
+ "help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 19
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 648,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in {{subreddit}} forum about what could large language models help the corresponding field.",
+ "instantiation_dict": {
+ "subreddit": "dataisbeautiful"
+ },
+ "intent": "Post in dataisbeautiful forum about what could large language models help the corresponding field.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/dataisbeautiful",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "large language models",
+ "help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 19
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 649,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Post in {{subreddit}} subreddit about what could diffusion model help the corresponding field.",
+ "instantiation_dict": {
+ "subreddit": "history"
+ },
+ "intent": "Post in history subreddit about what could diffusion model help the corresponding field.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/history",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "diffusion model",
+ "help |OR| assist |OR| benefit |OR| improve |OR| enhance |OR| transform |OR| revolutionize"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 19
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 650,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
+ "geolocation": null,
+ "intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
+ "instantiation_dict": {
+ "position_description": "the post",
+ "content_description": "I am a big fan of the bookorg"
+ },
+ "intent": "Reply to the post with my comment \"I am a big fan of the bookorg\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.comment__body').outerText",
+ "required_contents": {
+ "exact_match": "I am a big fan of the bookorg"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 23
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 651,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/pittsburgh/45899/driving-in-pittsburgh-summed-up-by-one-traffic-sign",
+ "geolocation": null,
+ "intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
+ "instantiation_dict": {
+ "position_description": "the post",
+ "content_description": "Yeah, pittsburgh traffic, you know..."
+ },
+ "intent": "Reply to the post with my comment \"Yeah, pittsburgh traffic, you know...\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/pittsburgh/45899/driving-in-pittsburgh-summed-up-by-one-traffic-sign",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.comment__body').outerText",
+ "required_contents": {
+ "exact_match": "Yeah, pittsburgh traffic, you know..."
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 23
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 652,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__/f/technology/134852/ai-experts-disown-musk-backed-campaign-citing-their-research",
+ "geolocation": null,
+ "intent_template": "Reply to {{position_description}} with my comment \"{{content_description}}\"",
+ "instantiation_dict": {
+ "position_description": "the post",
+ "content_description": "???"
+ },
+ "intent": "Reply to the post with my comment \"???\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/technology/134852/ai-experts-disown-musk-backed-campaign-citing-their-research",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.comment__body').outerText",
+ "required_contents": {
+ "exact_match": "???"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 23
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 653,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
+ "instantiation_dict": {
+ "product": "phone screen protector",
+ "order_id": "000000180"
+ },
+ "intent": "Fill the \"contact us\" form in the site for a refund on the phone screen protector I bought, stating that it broke after just three days of use. Also, ensure to include the order number #000000180 and the product SKU. Don't submit yet, I will check.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "000000180",
+ "B087QJN9W1"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 153
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 654,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
+ "instantiation_dict": {
+ "product": "bluetooth speaker",
+ "order_id": "161"
+ },
+ "intent": "Fill the \"contact us\" form in the site for a refund on the bluetooth speaker I bought, stating that it broke after just three days of use. Also, ensure to include the order number #161 and the product SKU. Don't submit yet, I will check.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "161",
+ "B09P7BFL4H"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 153
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 655,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
+ "instantiation_dict": {
+ "product": "iphone case",
+ "order_id": "180"
+ },
+ "intent": "Fill the \"contact us\" form in the site for a refund on the iphone case I bought, stating that it broke after just three days of use. Also, ensure to include the order number #180 and the product SKU. Don't submit yet, I will check.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "180",
+ "B087QJN9W1"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 153
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 656,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
+ "instantiation_dict": {
+ "product": "remote controller",
+ "order_id": "180"
+ },
+ "intent": "Fill the \"contact us\" form in the site for a refund on the remote controller I bought, stating that it broke after just three days of use. Also, ensure to include the order number #180 and the product SKU. Don't submit yet, I will check.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "180",
+ "B0041MSF2S"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 153
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 657,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Fill the \"contact us\" form in the site for a refund on the {{product}} I bought, stating that it broke after just three days of use. Also, ensure to include the order number #{{order_id}} and the product SKU. Don't submit yet, I will check.",
+ "instantiation_dict": {
+ "product": "speaker",
+ "order_id": "148"
+ },
+ "intent": "Fill the \"contact us\" form in the site for a refund on the speaker I bought, stating that it broke after just three days of use. Also, ensure to include the order number #148 and the product SKU. Don't submit yet, I will check.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "refund",
+ "broke",
+ "three days of use",
+ "148",
+ "B003FVW3VA"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 153
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 658,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
+ "instantiation_dict": {
+ "repo": "a11yproject",
+ "issue": "401 bad gateway",
+ "account": "Roshanjossey",
+ "due": "the end of 2030"
+ },
+ "intent": "Create an issue in a11yproject repo with title \"401 bad gateway\". Assign the issue to Roshanjossey. Set due date to be the end of 2030",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
+ "required_contents": {
+ "exact_match": "401 bad gateway"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Dec 31, 2030"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.assignee').outerText",
+ "required_contents": {
+ "must_include": [
+ "Roshan Jossey"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 327
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 659,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
+ "instantiation_dict": {
+ "repo": "empathy-prompts",
+ "issue": "Integrating LLMs for better prompts",
+ "account": "Roshanjossey",
+ "due": "the beginning of Q2 2033"
+ },
+ "intent": "Create an issue in empathy-prompts repo with title \"Integrating LLMs for better prompts\". Assign the issue to Roshanjossey. Set due date to be the beginning of Q2 2033",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/empathy-prompts/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
+ "required_contents": {
+ "exact_match": "Integrating LLMs for better prompts"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Apr 1, 2033"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.assignee').outerText",
+ "required_contents": {
+ "must_include": [
+ "Roshan Jossey"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 327
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 660,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
+ "instantiation_dict": {
+ "repo": "dotfiles",
+ "issue": "add support for oh-my-zsh",
+ "account": "Abishek",
+ "due": "July 18 2033"
+ },
+ "intent": "Create an issue in dotfiles repo with title \"add support for oh-my-zsh\". Assign the issue to Abishek. Set due date to be July 18 2033",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/dotfiles/-/tree/main",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
+ "required_contents": {
+ "exact_match": "add support for oh-my-zsh"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Jul 18, 2033"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.assignee').outerText",
+ "required_contents": {
+ "must_include": [
+ "Abishek S"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 327
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 661,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open an issue to {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "ChatGPT",
+ "issue": "report the issue of connection refused"
+ },
+ "intent": "Open an issue to report the issue of connection refused in ChatGPT.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/convexegg/chatgpt/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "connection refused"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 662,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open an issue to {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "aem-hacker",
+ "issue": "report experiencing \"OSError: [Errno 98] Address already in use\" during executions"
+ },
+ "intent": "Open an issue to report experiencing \"OSError: [Errno 98] Address already in use\" during executions in aem-hacker.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/0ang3el/aem-hacker/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "OSError: [Errno 98] Address already in use"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 663,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open an issue to {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "metaseq",
+ "issue": "ask their plan on supporting Llama and other llama family models"
+ },
+ "intent": "Open an issue to ask their plan on supporting Llama and other llama family models in metaseq.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/root/metaseq/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "llama"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 664,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open an issue to {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "awesome-python",
+ "issue": "ask their plans on adding Python 3.11 related resources"
+ },
+ "intent": "Open an issue to ask their plans on adding Python 3.11 related resources in awesome-python.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/vinta/awesome-python/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "Python 3.11"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 665,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Open an issue to {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "a11y-syntax-highlighting",
+ "issue": "request adding support for MT theme editor"
+ },
+ "intent": "Open an issue to request adding support for MT theme editor in a11y-syntax-highlighting.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/a11y-syntax-highlighting/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "MT theme editor"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 666,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Submit a request to merge {{source_branch}} branch into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "dialog-component",
+ "target_branch": "dialog",
+ "reviewer": "Carol"
+ },
+ "intent": "Submit a request to merge dialog-component branch into dialog branch, assign Carol as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "dialog"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "dialog-component"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Caroline Stewart"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 667,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "dialog-component",
+ "target_branch": "bump-doctocat",
+ "reviewer": "primer"
+ },
+ "intent": "Submit a merge request for dialog-component branch to be merged into bump-doctocat branch, assign primer as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "bump-doctocat"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "dialog-component"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Primer"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 668,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "a11yproject.com/redesign",
+ "target_branch": "master",
+ "reviewer": "Justin Armstrong"
+ },
+ "intent": "Submit a merge request for a11yproject.com/redesign branch to be merged into master branch, assign Justin Armstrong as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "redesign"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "main"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Justin Armstrong"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 669,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/solarized-prism-theme",
+ "geolocation": null,
+ "intent_template": "Open a new issue to discuss the implementation of {{feature}}",
+ "instantiation_dict": {
+ "feature": "dark mode"
+ },
+ "intent": "Open a new issue to discuss the implementation of dark mode",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/solarized-prism-theme/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "implementation",
+ "dark mode"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 337
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 670,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/byteblaze/dotfiles",
+ "geolocation": null,
+ "intent_template": "Open a new issue to discuss the implementation of {{feature}}",
+ "instantiation_dict": {
+ "feature": "default plugins for .zsh"
+ },
+ "intent": "Open a new issue to discuss the implementation of default plugins for .zsh",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/dotfiles/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.detail-page-description').outerText",
+ "required_contents": {
+ "must_include": [
+ "implementation",
+ "default plugins",
+ "zsh"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 337
+ },
+ {
+ "sites": [
+ "shopping",
+ "reddit"
+ ],
+ "task_id": 671,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
+ "instantiation_dict": {
+ "product": "Sony Computer Entertainment VR",
+ "rating": "2 stars and less"
+ },
+ "intent": "Gather the titles of Sony Computer Entertainment VR reviews with 2 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Sony Computer Entertainment VR\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__title').outerText",
+ "required_contents": {
+ "exact_match": "real user feedback on Sony Computer Entertainment VR"
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "didn't last a year without issues",
+ "Disappointing. Didn't last long before it stopped powering on and needed to be sent in for repair.",
+ "Received used items!!"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 101
+ },
+ {
+ "sites": [
+ "shopping",
+ "reddit"
+ ],
+ "task_id": 672,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
+ "instantiation_dict": {
+ "product": "Nintendo Switch Fortnite Wildcat Console EU",
+ "rating": "3 stars and less"
+ },
+ "intent": "Gather the titles of Nintendo Switch Fortnite Wildcat Console EU reviews with 3 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Nintendo Switch Fortnite Wildcat Console EU\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__title').outerText",
+ "required_contents": {
+ "exact_match": "real user feedback on Nintendo Switch Fortnite Wildcat Console EU"
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "EU charger and wild cat card doesn\u2019t even work!",
+ "REFUND REJECTED",
+ "Charging port not compatible",
+ "not compatible in the US",
+ "Wildcard Bonus Credits Not Redeemable!",
+ "Code not available!!"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 101
+ },
+ {
+ "sites": [
+ "shopping",
+ "reddit"
+ ],
+ "task_id": 673,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
+ "instantiation_dict": {
+ "product": "Racing Wheel Overdrive for Xbox X",
+ "rating": "1 star"
+ },
+ "intent": "Gather the titles of Racing Wheel Overdrive for Xbox X reviews with 1 star rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Racing Wheel Overdrive for Xbox X\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__title').outerText",
+ "required_contents": {
+ "exact_match": "real user feedback on Racing Wheel Overdrive for Xbox X"
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "Unable to set neutral steering",
+ "Doesn\u2019t work with PC",
+ "Crazy problems in automatic mode",
+ "pedals stopped working",
+ "Only works with certain games"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 101
+ },
+ {
+ "sites": [
+ "shopping",
+ "reddit"
+ ],
+ "task_id": 674,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
+ "instantiation_dict": {
+ "product": "Doc and Pies Arcade Factory Cocktail Arcade Machine",
+ "rating": "3 stars and less"
+ },
+ "intent": "Gather the titles of Doc and Pies Arcade Factory Cocktail Arcade Machine reviews with 3 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on Doc and Pies Arcade Factory Cocktail Arcade Machine\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__title').outerText",
+ "required_contents": {
+ "exact_match": "real user feedback on Doc and Pies Arcade Factory Cocktail Arcade Machine"
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "Poorly Made Exterior. Consider a different Company.",
+ "piece of junk ,..can't believe I spent money on this !!!!",
+ "Based arrived broken but game itself works"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 101
+ },
+ {
+ "sites": [
+ "shopping",
+ "reddit"
+ ],
+ "task_id": 675,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Gather the titles of {{product}} reviews with {{rating}} rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on {{product}}\"",
+ "instantiation_dict": {
+ "product": "HORI 3D Surround Gaming Neckset",
+ "rating": "2 stars and less"
+ },
+ "intent": "Gather the titles of HORI 3D Surround Gaming Neckset reviews with 2 stars and less rating from OneStopShop, and post them in the games subreddit under the title \"real user feedback on HORI 3D Surround Gaming Neckset\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/gaming",
+ "program_html": [
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__title').outerText",
+ "required_contents": {
+ "exact_match": "real user feedback on HORI 3D Surround Gaming Neckset"
+ }
+ },
+ {
+ "url": "func:reddit_get_post_url('__last_url__')",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "Not worth it for PC users",
+ "I really wanted to like this.",
+ "I wish this was better..."
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 101
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 676,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Lookup orders that are {{status}}",
+ "instantiation_dict": {
+ "status": "suspected of being fraudulent"
+ },
+ "intent": "Lookup orders that are suspected of being fraudulent",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales/order/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
+ "required_contents": {
+ "must_include": [
+ "Suspected Fraud"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 253
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 677,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Lookup orders that are {{status}}",
+ "instantiation_dict": {
+ "status": "processing"
+ },
+ "intent": "Lookup orders that are processing",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales/order/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
+ "required_contents": {
+ "must_include": [
+ "Processing"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 253
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 678,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Lookup orders that are {{status}}",
+ "instantiation_dict": {
+ "status": "canceled"
+ },
+ "intent": "Lookup orders that are canceled",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales/order/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
+ "required_contents": {
+ "must_include": [
+ "Canceled"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 253
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 679,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Lookup orders that are {{status}}",
+ "instantiation_dict": {
+ "status": "completed"
+ },
+ "intent": "Lookup orders that are completed",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales/order/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
+ "required_contents": {
+ "must_include": [
+ "Complete"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 253
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 680,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Lookup orders that are {{status}}",
+ "instantiation_dict": {
+ "status": "on hold"
+ },
+ "intent": "Lookup orders that are on hold",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales/order/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div.admin__data-grid-filters-current\").outerText",
+ "required_contents": {
+ "must_include": [
+ "On Hold"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 253
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 681,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
+ "instantiation_dict": {
+ "topic": "gan implementation"
+ },
+ "intent": "Find a GitLab repository related to gan implementation and make a Reddit post linking to it in a relevant subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning |OR| __REDDIT__/f/technology",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "gan implementation",
+ "__GITLAB__/eriklindernoren/PyTorch-GAN"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 116
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 682,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
+ "instantiation_dict": {
+ "topic": "chatGPT"
+ },
+ "intent": "Find a GitLab repository related to chatGPT and make a Reddit post linking to it in a relevant subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/singularity |OR| __REDDIT__/f/deeplearning |OR| __REDDIT__/f/technology",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "chatgpt",
+ "__GITLAB__/convexegg/chatgpt"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 116
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 683,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Find a GitLab repository related to {{topic}} and make a Reddit post linking to it in a relevant subreddit",
+ "instantiation_dict": {
+ "topic": "metaseq"
+ },
+ "intent": "Find a GitLab repository related to metaseq and make a Reddit post linking to it in a relevant subreddit",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/machinelearning |OR| __REDDIT__/f/deeplearning",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText",
+ "required_contents": {
+ "must_include": [
+ "metaseq",
+ "__GITLAB__/root/metaseq"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 116
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 684,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
+ "instantiation_dict": {
+ "repo": "byteblaze/cloud-to-butt",
+ "subreddit": "LifeProTips"
+ },
+ "intent": "Promote byteblaze/cloud-to-butt to subreddit LifeProTips with the description from the repo itself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/LifeProTips",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "Chrome extension that replaces occurrences of 'the cloud' with 'my butt'",
+ "__GITLAB__/byteblaze/cloud-to-butt"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 117
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 685,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
+ "instantiation_dict": {
+ "repo": "byteblaze/dotfiles",
+ "subreddit": "aww"
+ },
+ "intent": "Promote byteblaze/dotfiles to subreddit aww with the description from the repo itself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/aww",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "Computer setup",
+ "__GITLAB__/byteblaze/dotfiles"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 117
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 686,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
+ "instantiation_dict": {
+ "repo": "auth0/angular-storage",
+ "subreddit": "technology"
+ },
+ "intent": "Promote auth0/angular-storage to subreddit technology with the description from the repo itself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/technology",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "A storage library for AngularJS done right",
+ "__GITLAB__/auth0/angular-storage"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 117
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 687,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
+ "instantiation_dict": {
+ "repo": "koush/AndroidAsync",
+ "subreddit": "funny"
+ },
+ "intent": "Promote koush/AndroidAsync to subreddit funny with the description from the repo itself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/funny",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "Asynchronous socket, http(s) (client+server) and websocket library for android. Based on nio, not threads.",
+ "__GITLAB__/koush/AndroidAsync"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 117
+ },
+ {
+ "sites": [
+ "reddit",
+ "gitlab"
+ ],
+ "task_id": 688,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Promote {{repo}} to subreddit {{subreddit}} with the description from the repo itself.",
+ "instantiation_dict": {
+ "repo": "lahwaacz/arch-wiki-docs",
+ "subreddit": "science"
+ },
+ "intent": "Promote lahwaacz/arch-wiki-docs to subreddit science with the description from the repo itself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__REDDIT__/f/science",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('.submission__inner').outerText + [...document.querySelector('.submission__inner').querySelectorAll('[href],[src]')].map(elem => elem.getAttribute('href') || elem.getAttribute('src')).join(' ')",
+ "required_contents": {
+ "must_include": [
+ "A script to download pages from Arch Wiki for offline browsing",
+ "__GITLAB__/lahwaacz/arch-wiki-docs"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 117
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 689,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
+ "instantiation_dict": {
+ "reason": "I am a loyal customer"
+ },
+ "intent": "Draft an email to the shop owner via their contact us function for a coupon as I am a loyal customer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "coupon",
+ "I am a loyal customer"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 163
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 690,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
+ "instantiation_dict": {
+ "reason": "they promised me a coupon last time"
+ },
+ "intent": "Draft an email to the shop owner via their contact us function for a coupon as they promised me a coupon last time",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "coupon",
+ "promised"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 163
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 691,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
+ "instantiation_dict": {
+ "reason": "I plan to make a bulk purchase"
+ },
+ "intent": "Draft an email to the shop owner via their contact us function for a coupon as I plan to make a bulk purchase",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "coupon",
+ "bulk purchase"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 163
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 692,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
+ "instantiation_dict": {
+ "reason": "I am a student"
+ },
+ "intent": "Draft an email to the shop owner via their contact us function for a coupon as I am a student",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "coupon",
+ "student"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 163
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 693,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Draft an email to the shop owner via their contact us function for a coupon as {{reason}}",
+ "instantiation_dict": {
+ "reason": "my refund is suppoed to be replaced by a coupon"
+ },
+ "intent": "Draft an email to the shop owner via their contact us function for a coupon as my refund is suppoed to be replaced by a coupon",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING__/contact",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[title=\"What\u2019s on your mind?\"').value",
+ "required_contents": {
+ "must_include": [
+ "coupon",
+ "refund"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 163
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 694,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
+ "instantiation_dict": {
+ "product": "Energy-Bulk Women Shirt",
+ "stock": "50",
+ "size": "S",
+ "color": "blue",
+ "price": "60"
+ },
+ "intent": "Add a simple product named Energy-Bulk Women Shirt with 50 in stock, available in size S and color blue, priced at $60",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/catalog/product",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "60.00"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[name]\"').value",
+ "required_contents": {
+ "must_include": [
+ "Energy-Bulk Women Shirt"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "50"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "top"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[size]\"').value",
+ "required_contents": {
+ "exact_match": "167"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[color]\"').value",
+ "required_contents": {
+ "exact_match": "50"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "tops"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 256
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 695,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
+ "instantiation_dict": {
+ "product": "Energy-Bulk Man Yoga Pant",
+ "stock": "50",
+ "size": "38",
+ "color": "yellow",
+ "price": "69.99"
+ },
+ "intent": "Add a simple product named Energy-Bulk Man Yoga Pant with 50 in stock, available in size 38 and color yellow, priced at $69.99",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/catalog/product",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "69.99"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[name]\"').value",
+ "required_contents": {
+ "must_include": [
+ "Energy-Bulk Man Yoga Pant"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "50"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "bottom"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[size]\"').value",
+ "required_contents": {
+ "exact_match": "179"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[color]\"').value",
+ "required_contents": {
+ "exact_match": "60"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "bottoms"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 256
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 696,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
+ "instantiation_dict": {
+ "product": "FancyBoy Man Causal Jeans",
+ "stock": "42",
+ "size": "34",
+ "color": "Blue",
+ "price": "169.99"
+ },
+ "intent": "Add a simple product named FancyBoy Man Causal Jeans with 42 in stock, available in size 34 and color Blue, priced at $169.99",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/catalog/product",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[name]\"').value",
+ "required_contents": {
+ "must_include": [
+ "FancyBoy Man Causal Jeans"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "42"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "169.99"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "bottom"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[size]\"').value",
+ "required_contents": {
+ "exact_match": "177"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[color]\"').value",
+ "required_contents": {
+ "exact_match": "50"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "bottoms"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 256
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 697,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
+ "instantiation_dict": {
+ "product": "Swaatch Smart Watch",
+ "stock": "42",
+ "size": "uni-size",
+ "color": "Blue",
+ "price": "769.99"
+ },
+ "intent": "Add a simple product named Swaatch Smart Watch with 42 in stock, available in size uni-size and color Blue, priced at $769.99",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/catalog/product",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[name]\"').value",
+ "required_contents": {
+ "must_include": [
+ "Swaatch Smart Watch"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "42"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "769.99"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "gear"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[color]\"').value",
+ "required_contents": {
+ "exact_match": "50"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "watches"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 256
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 698,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Add a simple product named {{product}} with {{stock}} in stock, available in size {{size}} and color {{color}}, priced at ${{price}}",
+ "instantiation_dict": {
+ "product": "Lelelumon Yoga Mat",
+ "stock": "42",
+ "size": "uni-size",
+ "color": "black",
+ "price": "769.99"
+ },
+ "intent": "Add a simple product named Lelelumon Yoga Mat with 42 in stock, available in size uni-size and color black, priced at $769.99",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/catalog/product",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[name]\"').value",
+ "required_contents": {
+ "must_include": [
+ "Lelelumon Yoga Mat"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "42"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "769.99"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-role=\"selected-option\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "gear"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"product[color]\"').value",
+ "required_contents": {
+ "exact_match": "49"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-index=\"category_ids\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "fitness equipment"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 256
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 699,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
+ "instantiation_dict": {
+ "topic": "spring sale",
+ "rule": "a 20 percent discount site-wide"
+ },
+ "intent": "Draft a new marketing price rule for spring sale that offers a 20 percent discount site-wide for all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"[name='name'\").value",
+ "required_contents": {
+ "must_include": [
+ "spring sale"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"simple_action\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "by_percent"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"discount_amount\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "20"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 258
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 700,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
+ "instantiation_dict": {
+ "topic": "fall discount",
+ "rule": "$10 discount on checkout"
+ },
+ "intent": "Draft a new marketing price rule for fall discount that offers $10 discount on checkout for all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"[name='name'\").value",
+ "required_contents": {
+ "must_include": [
+ "fall discount"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"simple_action\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "cart_fixed"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"discount_amount\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "10"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 258
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 701,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
+ "instantiation_dict": {
+ "topic": "Mother's day sale",
+ "rule": "$15 discount on checkout"
+ },
+ "intent": "Draft a new marketing price rule for Mother's day sale that offers $15 discount on checkout for all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"[name='name'\").value",
+ "required_contents": {
+ "must_include": [
+ "Mother's day sale"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"simple_action\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "cart_fixed"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"discount_amount\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "15"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 258
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 702,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
+ "instantiation_dict": {
+ "topic": "Pride Month",
+ "rule": "45% off on all products"
+ },
+ "intent": "Draft a new marketing price rule for Pride Month that offers 45% off on all products for all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"[name='name'\").value",
+ "required_contents": {
+ "must_include": [
+ "Pride Month"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"simple_action\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "by_percent"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"discount_amount\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "45"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 258
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 703,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Draft a new marketing price rule for {{topic}} that offers {{rule}} for all customers",
+ "instantiation_dict": {
+ "topic": "Thanks giving sale",
+ "rule": "$40 discount on checkout"
+ },
+ "intent": "Draft a new marketing price rule for Thanks giving sale that offers $40 discount on checkout for all customers",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/sales_rule/promo_quote",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"[name='name'\").value",
+ "required_contents": {
+ "must_include": [
+ "Thanks giving sale"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"website_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "0"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"customer_group_ids\"').selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"simple_action\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "cart_fixed"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"discount_amount\"').value",
+ "prep_actions": ["document.querySelector('[data-index=\"actions\"]').querySelector('.admin__collapsible-title').click()"],
+ "required_contents": {
+ "exact_match": "40"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 258
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 704,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
+ "instantiation_dict": {
+ "report": "sales order report",
+ "time_span": "for last month"
+ },
+ "intent": "Today is 3/15/2023, generate a sales order report for last month",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "2/1/23"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "2/28/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 268
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 705,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
+ "instantiation_dict": {
+ "report": "sales order report",
+ "time_span": "over the last 45 days"
+ },
+ "intent": "Today is 3/15/2023, generate a sales order report over the last 45 days",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "1/29/23"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "3/15/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 268
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 706,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
+ "instantiation_dict": {
+ "report": "refund report",
+ "time_span": "for Q1"
+ },
+ "intent": "Today is 3/15/2023, generate a refund report for Q1",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/refunded",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "1/1/23"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "3/31/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 268
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 707,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
+ "instantiation_dict": {
+ "report": "sales order report",
+ "time_span": "for last year"
+ },
+ "intent": "Today is 3/15/2023, generate a sales order report for last year",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "1/1/22"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "12/31/22"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 268
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 708,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Today is 3/15/2023, generate a {{report}} {{time_span}}",
+ "instantiation_dict": {
+ "report": "tax report",
+ "time_span": "for this year"
+ },
+ "intent": "Today is 3/15/2023, generate a tax report for this year",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/tax/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "1/1/23"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "12/31/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 268
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 709,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Create an {{type}} report from {{start_date}} to {{end_date}}",
+ "instantiation_dict": {
+ "type": "orders",
+ "start_date": "beginning of May 2021",
+ "end_date": "end of March 2022"
+ },
+ "intent": "Create an orders report from beginning of May 2021 to end of March 2022",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/sales",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "5/1/21"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "3/31/22"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 271
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 710,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
+ "instantiation_dict": {
+ "type": "shipping",
+ "start_date": "08/05/2022",
+ "end_date": "03/01/2023"
+ },
+ "intent": "Create a shipping report from 08/05/2022 to 03/01/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/shipping",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "8/5/22"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "3/1/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 271
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 711,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
+ "instantiation_dict": {
+ "type": "product view",
+ "start_date": "07/05/2021",
+ "end_date": "05/31/2023"
+ },
+ "intent": "Create a product view report from 07/05/2021 to 05/31/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_product/viewed/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "7/5/21"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "5/31/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 271
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 712,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
+ "instantiation_dict": {
+ "type": "coupons",
+ "start_date": "05/01/2021",
+ "end_date": "05/15/2023"
+ },
+ "intent": "Create a coupons report from 05/01/2021 to 05/15/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/coupons/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "5/1/21"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "5/15/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 271
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 713,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Create a {{type}} report from {{start_date}} to {{end_date}}",
+ "instantiation_dict": {
+ "type": "best sellers",
+ "start_date": "05/01/2022",
+ "end_date": "05/31/2023"
+ },
+ "intent": "Create a best sellers report from 05/01/2022 to 05/31/2023",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__SHOPPING_ADMIN__/reports/report_sales/bestsellers/",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_from\"').value",
+ "required_contents": {
+ "exact_match": "5/1/22"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[id=\"sales_report_to\"').value",
+ "required_contents": {
+ "exact_match": "5/31/23"
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 271
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 714,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
+ "instantiation_dict": {
+ "subreddit": "gadgets",
+ "k": "1"
+ },
+ "intent": "Thumbs down the top 1 post ever in gadgets.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/gadgets/19459/a-custom-gaming-pc-built-inside-a-vintage-1940s-motorola",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 24
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 715,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
+ "instantiation_dict": {
+ "subreddit": "history",
+ "k": "2"
+ },
+ "intent": "Thumbs down the top 2 post ever in history.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/history/84338/the-scientist-who-discovered-sperm-was-so-grossed-out-he",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/history/105990/4-500-year-old-sumerian-temple-dedicated-to-mighty-thunder",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 24
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 716,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
+ "instantiation_dict": {
+ "k": "3",
+ "subreddit": "books"
+ },
+ "intent": "Thumbs down the top 3 post ever in books.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/books/81371/the-letters-of-t-s-eliot-to-emily-hale-that-were-kept-sealed",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/books/59421/friendly-reminder-bookshop-org-exists",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/books/59447/appalachian-prison-book-project-seeks-notebook-donations-the",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 24
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 717,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
+ "instantiation_dict": {
+ "k": "4",
+ "subreddit": "movies"
+ },
+ "intent": "Thumbs down the top 4 post ever in movies.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/movies/86174/who-will-win-the-oscar-for-actress-in-a-supporting-role",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/movies/86029/who-will-win-the-oscar-for-film-editing",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/movies/86055/cindy-williams-dies-laverne-amp-shirley-star-who-appeared-in",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/movies/42682/michelle-yeoh-to-receive-palm-springs-film-festival-s",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 24
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 718,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Thumbs down the top {{k}} post ever in {{subreddit}}.",
+ "instantiation_dict": {
+ "k": "5",
+ "subreddit": "technology"
+ },
+ "intent": "Thumbs down the top 5 post ever in technology.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/technology/48670/brain-cancer-vaccine-succeeds-at-prolonging-survival-in",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/technology/134696/india-cuts-internet-for-27-million-people-amid-search-for",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/technology/48785/us-judge-orders-amazon-to-cease-and-desist-anti-union",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/technology/70354/activision-s-boston-studio-workers-announce-unionization",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/technology/70233/social-media-influencers-are-charged-with-feeding-followers",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 24
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 719,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "subreddit": "wallstreetbets",
+ "user": "ThetaGang_wsb"
+ },
+ "intent": "Like all submissions created by ThetaGang_wsb in subreddit wallstreetbets",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/wallstreetbets/29478/how-will-airbnb-close-following-their-earnings-report-on",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/wallstreetbets/29458/how-much-will-the-federal-reserve-raise-interest-rates-in",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 25
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 720,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "subreddit": "earthporn",
+ "user": "CameronKelsey"
+ },
+ "intent": "Like all submissions created by CameronKelsey in subreddit earthporn",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/EarthPorn/98332/my-favorite-place-on-the-planet-henry-s-fork-of-the-snake",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/EarthPorn/98297/2-years-later-this-is-still-one-of-the-most-incredible",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/EarthPorn/98256/i-can-t-wait-for-all-this-green-to-start-coming-back-little",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 25
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 721,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "UniversityofBath",
+ "subreddit": "IAmA"
+ },
+ "intent": "Like all submissions created by UniversityofBath in subreddit IAmA",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/IAmA/119742/hi-i-m-vienne-a-doctoral-student-at-the-university-of-bath-i",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/119719/hello-reddit-i-m-nazia-mehrban-a-lecturer-in-biotechnology",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/119714/i-m-ellie-jarvis-she-her-a-2nd-year-phd-student-in-the",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/55155/hi-i-m-dr-lucy-maddox-from-bath-university-uk-i-m-a-clinical",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/55142/we-re-sadeka-nujhat-hannah-leese-and-sandhya-moise-from-the",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/34032/we-re-sandhya-moise-david-phillips-and-chan-lee-from-the",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/13175/hi-i-m-kit-yates-i-m-a-mathematical-biologist-at-the",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/IAmA/13170/hello-i-m-dr-sara-fontani-from-the-university-of",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 25
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 722,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "Don_Gato1",
+ "subreddit": "new york"
+ },
+ "intent": "Like all submissions created by Don_Gato1 in subreddit new york",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/nyc/44650/fox-news-hosts-cast-new-york-as-crime-ridden-and-chaotic",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 25
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 723,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "FTorrez81",
+ "subreddit": "iphone13"
+ },
+ "intent": "Like all submissions created by FTorrez81 in subreddit iphone13",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "reference_answer_raw_annotation": "N/A",
+ "string_note": "FTorrez81 does not have any submissions in iphone13"
+ },
+ "intent_template_id": 25,
+ "string_note": "FTorrez81 has no submissions in subreddit iphone13"
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 724,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Like all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "Hrekires",
+ "subreddit": "news"
+ },
+ "intent": "Like all submissions created by Hrekires in subreddit news",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/news/129816/gov-whitmer-signs-bills-to-repeal-right-to-work-restore",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129808/disney-world-deal-with-union-will-raise-minimum-wage-to-18",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129794/judge-halts-wyoming-abortion-ban-days-after-it-took-effect",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129783/don-t-say-gay-lawmaker-pleads-guilty-to-covid-relief-fraud",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129594/arizona-gov-katie-hobbs-refuses-to-proceed-with-execution",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129508/tennessee-governor-oks-bill-to-cut-nashville-council-in-half",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43839/philadelphia-da-larry-krasner-impeached-by-pa-house",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43781/crypto-giant-ftx-to-file-for-bankruptcy-ceo-sam-bankman",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43572/sec-doj-investigating-crypto-platform-ftx",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43558/kansas-gov-laura-kelly-wins-re-election-defeating-gop",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-upvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 25
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 725,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "subreddit": "massachusetts",
+ "user": "RickyDontLoseThat"
+ },
+ "intent": "DisLike all submissions created by RickyDontLoseThat in subreddit massachusetts",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/massachusetts/84954/the-last-of-lincoln",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 726,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "subreddit": "earthporn",
+ "user": "jacyanthis"
+ },
+ "intent": "DisLike all submissions created by jacyanthis in subreddit earthporn",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "jacyanthis does not have any submissions in earthporn",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 727,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "PatientBuilder499",
+ "subreddit": "videos"
+ },
+ "intent": "DisLike all submissions created by PatientBuilder499 in subreddit videos",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/videos/115139/hundreds-of-civilian-turkish-volunteers-waiting-to-be-sent",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 728,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "sirbarani",
+ "subreddit": "sports"
+ },
+ "intent": "DisLike all submissions created by sirbarani in subreddit sports",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/sports/48303/iran-football-legend-daei-will-not-attend-world-cup-amid",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 729,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "AdamCannon",
+ "subreddit": "UpliftingNews"
+ },
+ "intent": "DisLike all submissions created by AdamCannon in subreddit UpliftingNews",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/UpliftingNews/16087/same-sex-marriage-is-now-legal-in-all-of-mexico-s-states",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 730,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "DisLike all submissions created by {{user}} in subreddit {{subreddit}}",
+ "instantiation_dict": {
+ "user": "Hrekires",
+ "subreddit": "news"
+ },
+ "intent": "DisLike all submissions created by Hrekires in subreddit news",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/news/129816/gov-whitmer-signs-bills-to-repeal-right-to-work-restore",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129808/disney-world-deal-with-union-will-raise-minimum-wage-to-18",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129794/judge-halts-wyoming-abortion-ban-days-after-it-took-effect",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129783/don-t-say-gay-lawmaker-pleads-guilty-to-covid-relief-fraud",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129594/arizona-gov-katie-hobbs-refuses-to-proceed-with-execution",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/129508/tennessee-governor-oks-bill-to-cut-nashville-council-in-half",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43839/philadelphia-da-larry-krasner-impeached-by-pa-house",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43781/crypto-giant-ftx-to-file-for-bankruptcy-ceo-sam-bankman",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43572/sec-doj-investigating-crypto-platform-ftx",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ },
+ {
+ "url": "__REDDIT__/f/news/43558/kansas-gov-laura-kelly-wins-re-election-defeating-gop",
+ "locator": "document.querySelector('div.submission__vote').querySelector('form').getAttribute('class')",
+ "required_contents": {
+ "must_include": [
+ "vote vote--user-downvoted"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 1510
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 731,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
+ "instantiation_dict": {
+ "post": "Nvidia RTX 4090",
+ "content": "EDIT: This news aged well"
+ },
+ "intent": "Edit my post on Nvidia RTX 4090 by adding a line to the body that says \"EDIT: This news aged well\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/MachineLearning/1/nvidia-rtx-4090",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "must_include": [
+ "EDIT: This news aged well",
+ "Crazy device for ML!"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 27
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 732,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
+ "instantiation_dict": {
+ "post": "The Night Agent",
+ "content": "Done watching, pretty cool!"
+ },
+ "intent": "Edit my post on The Night Agent by adding a line to the body that says \"Done watching, pretty cool!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/television/134868/the-night-agent-renewed-for-season-2-at-netflix",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "exact_match": "Done watching, pretty cool!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 27
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 733,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
+ "instantiation_dict": {
+ "post": "Star Trek Starfleet Academy series",
+ "content": "Every watch makes me feel like a kid again"
+ },
+ "intent": "Edit my post on Star Trek Starfleet Academy series by adding a line to the body that says \"Every watch makes me feel like a kid again\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/television/135201/star-trek-starfleet-academy-series-from-alex-kurtzman-and",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "exact_match": "Every watch makes me feel like a kid again"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 27
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 734,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
+ "instantiation_dict": {
+ "post": "Ted Lasso",
+ "content": "Done watching. I love the renew!"
+ },
+ "intent": "Edit my post on Ted Lasso by adding a line to the body that says \"Done watching. I love the renew!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/television/135156/ted-lasso-season-3-premiere-scores-870k-u-s-households-up-59",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "exact_match": "Done watching. I love the renew!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 27
+ },
+ {
+ "sites": [
+ "reddit"
+ ],
+ "task_id": 735,
+ "require_login": true,
+ "storage_state": "./.auth/reddit_state.json",
+ "start_url": "__REDDIT__",
+ "geolocation": null,
+ "intent_template": "Edit my post on {{post}} by adding a line to the body that says \"{{content}}\"",
+ "instantiation_dict": {
+ "post": "Lord of the Rings",
+ "content": "The cast is amazing!"
+ },
+ "intent": "Edit my post on Lord of the Rings by adding a line to the body that says \"The cast is amazing!\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__REDDIT__/f/television/135152/lord-of-the-rings-the-rings-of-power-season-2-cast-adds",
+ "locator": "document.querySelector('.submission__body').outerText",
+ "required_contents": {
+ "exact_match": "The cast is amazing!"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 27
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 736,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make the LICENSE of {{repo}} to MIT license.",
+ "instantiation_dict": {
+ "repo": "gimmiethat.space and dotfiles"
+ },
+ "intent": "Make the LICENSE of gimmiethat.space and dotfiles to MIT license.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/gimmiethat.space/-/blob/main/LICENSE",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "MIT license",
+ "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/dotfiles/-/blob/main/LICENSE",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "MIT license",
+ "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software."
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 355
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 737,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University",
+ "sport_team": "Philadelphia 76ers",
+ "time": ""
+ },
+ "intent": "Show me the way from Carnegie Mellon University to the home stadium of Philadelphia 76ers ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Wells Fargo Center",
+ "South Philadelphia Sports Complex"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 94
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 738,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University",
+ "sport_team": "Philadelphia 76ers",
+ "time": "in the 70th"
+ },
+ "intent": "Show me the way from Carnegie Mellon University to the home stadium of Philadelphia 76ers in the 70th",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "3601 South Broad Street",
+ "South Philadelphia"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 94
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 739,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University",
+ "sport_team": "Yankees",
+ "time": "in the 80th"
+ },
+ "intent": "Show me the way from Carnegie Mellon University to the home stadium of Yankees in the 80th",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Yankee Stadium",
+ "East 161st Street"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 94
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 740,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University",
+ "sport_team": "NYC NBA team",
+ "time": ""
+ },
+ "intent": "Show me the way from Carnegie Mellon University to the home stadium of NYC NBA team ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Madison Square Garden",
+ "Pennsylvania Plaza",
+ "Manhattan",
+ "New York"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 94
+ },
+ {
+ "sites": [
+ "wikipedia",
+ "map"
+ ],
+ "task_id": 741,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the way from {{location}} to the home stadium of {{sport_team}} {{time}}",
+ "instantiation_dict": {
+ "location": "Carnegie Mellon University",
+ "sport_team": "Boston home NBA team",
+ "time": ""
+ },
+ "intent": "Show me the way from Carnegie Mellon University to the home stadium of Boston home NBA team ",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "TD Garden",
+ "Boston",
+ "Massachusetts"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 94
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 742,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a new {{scope}} project \"planner\" and add {{account_list}} as members",
+ "instantiation_dict": {
+ "scope": "private",
+ "account_list": "Abishek, Vinta"
+ },
+ "intent": "Create a new private project \"planner\" and add Abishek, Vinta as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/planner",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/planner/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@abisubramanya27",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 743,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a new {{scope}} project \"web_arena\" and add {{account_list}} as members",
+ "instantiation_dict": {
+ "scope": "public",
+ "account_list": "Abishek, Vinta"
+ },
+ "intent": "Create a new public project \"web_arena\" and add Abishek, Vinta as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_arena",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "public"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_arena/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@abisubramanya27",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 744,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a new {{scope}} project \"AutoAGI\" and add {{account_list}} as members",
+ "instantiation_dict": {
+ "scope": "public",
+ "account_list": "primer"
+ },
+ "intent": "Create a new public project \"AutoAGI\" and add primer as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/AutoAGI",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "public"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/AutoAGI/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@primer"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 745,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a new {{scope}} project \"awesome-llms\" and add {{account_list}} as members",
+ "instantiation_dict": {
+ "scope": "public",
+ "account_list": "primer, convexegg, abishek"
+ },
+ "intent": "Create a new public project \"awesome-llms\" and add primer, convexegg, abishek as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/awesome-llms",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "public"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/awesome-llms/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@primer",
+ "@convexegg",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 746,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a new {{scope}} project \"llm_bulk_inference\" and add {{account_list}} as members",
+ "instantiation_dict": {
+ "scope": "private",
+ "account_list": "primer, convexegg, abishek"
+ },
+ "intent": "Create a new private project \"llm_bulk_inference\" and add primer, convexegg, abishek as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/llm_bulk_inference",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/llm_bulk_inference/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@primer",
+ "@convexegg",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 747,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
+ "instantiation_dict": {
+ "project_name": "awesome_web_agents",
+ "template": "blank",
+ "account_list": "Abishek, Vinta"
+ },
+ "intent": "Start a private project awesome_web_agents with blank template and add Abishek, Vinta as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/awesome_web_agents",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/awesome_web_agents/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initial commit"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/awesome_web_agents/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@abisubramanya27",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 2100
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 748,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
+ "instantiation_dict": {
+ "project_name": "web_agent_android_xl",
+ "template": "Android",
+ "account_list": "primer, convexegg, abishek"
+ },
+ "intent": "Start a private project web_agent_android_xl with Android template and add primer, convexegg, abishek as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_android_xl",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_android_xl/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initialized from 'Android' project template"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_android_xl/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@primer",
+ "@convexegg",
+ "@abisubramanya27"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 2100
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 749,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
+ "instantiation_dict": {
+ "project_name": "project_site",
+ "template": "NodeJS",
+ "account_list": "primer, convexegg, vinta"
+ },
+ "intent": "Start a private project project_site with NodeJS template and add primer, convexegg, vinta as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/project_site",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/project_site/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initialized from 'NodeJS Express' project template"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/project_site/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@primer",
+ "@convexegg",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 2100
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 750,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
+ "instantiation_dict": {
+ "project_name": "agi_index",
+ "template": "HTML",
+ "account_list": "Vinta Chen"
+ },
+ "intent": "Start a private project agi_index with HTML template and add Vinta Chen as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/agi_index",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/agi_index",
+ "locator": "document.querySelector('.home-panel-description-markdown').outerText",
+ "required_contents": {
+ "must_include": [
+ "Example plain HTML site using GitLab Pages: https://pages.gitlab.io/plain-html |OR| A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/agi_index/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Vinta Chen"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 2100
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 751,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Start a private project {{project_name}} with {{template}} template and add {{account_list}} as members",
+ "instantiation_dict": {
+ "project_name": "AGISite",
+ "template": "JEKYLL",
+ "account_list": "Rohan and Vinta"
+ },
+ "intent": "Start a private project AGISite with JEKYLL template and add Rohan and Vinta as members",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/AGISite",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/AGISite",
+ "locator": "document.querySelector('.home-panel-description-markdown').outerText",
+ "required_contents": {
+ "must_include": [
+ "Example Jekyll site using GitLab Pages: https://pages.gitlab.io/jekyll |OR| A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/AGISite/-/project_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@Seirdy",
+ "@vinta"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 2100
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 752,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
+ "instantiation_dict": {
+ "project_name": "web_agent",
+ "template": "blank"
+ },
+ "intent": "Create a private blank repository called \"web_agent\" using the right template to speed up development.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_agent",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initial commit"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 753,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
+ "instantiation_dict": {
+ "project_name": "web_agent_android_xs",
+ "template": "Android"
+ },
+ "intent": "Create a private Android repository called \"web_agent_android_xs\" using the right template to speed up development.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_android_xs",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_android_xs/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initialized from 'Android' project template"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 754,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
+ "instantiation_dict": {
+ "project_name": "web_agent_nodejs",
+ "template": "NodeJS"
+ },
+ "intent": "Create a private NodeJS repository called \"web_agent_nodejs\" using the right template to speed up development.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_nodejs",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_nodejs/-/commits",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Initialized from 'NodeJS Express' project template"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 755,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
+ "instantiation_dict": {
+ "project_name": "web_agent_index",
+ "template": "HTML"
+ },
+ "intent": "Create a private HTML repository called \"web_agent_index\" using the right template to speed up development.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_index",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/web_agent_index",
+ "locator": "document.querySelector('.home-panel-description-markdown').outerText",
+ "required_contents": {
+ "must_include": [
+ "Example plain HTML site using GitLab Pages: https://pages.gitlab.io/plain-html |OR| A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 756,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create a private {{template}} repository called \"{{project_name}}\" using the right template to speed up development.",
+ "instantiation_dict": {
+ "project_name": "11711_gitlab",
+ "template": "JEKYLL"
+ },
+ "intent": "Create a private JEKYLL repository called \"11711_gitlab\" using the right template to speed up development.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/byteblaze/11711_gitlab",
+ "locator": "document.querySelector('.visibility-icon').getAttribute('title')",
+ "required_contents": {
+ "must_include": [
+ "Private"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/byteblaze/11711_gitlab",
+ "locator": "document.querySelector('.home-panel-description-markdown').outerText",
+ "required_contents": {
+ "must_include": [
+ "Example Jekyll site using GitLab Pages: https://pages.gitlab.io/jekyll |OR| A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features."
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 332
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 757,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the path and travel time from {{city1}} to {{city2}}.",
+ "instantiation_dict": {
+ "city1": "home of the 1980 Super Bowl champions",
+ "city2": "home of the 1991 Super Bowl champions"
+ },
+ "intent": "Show me the path and travel time from home of the 1980 Super Bowl champions to home of the 1991 Super Bowl champions.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "New York"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 42
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 758,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the path and travel time from {{city1}} to {{city2}}.",
+ "instantiation_dict": {
+ "city1": "the big apple",
+ "city2": "biggest city in Maine"
+ },
+ "intent": "Show me the path and travel time from the big apple to biggest city in Maine.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "New York"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Portland",
+ "Maine"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 42
+ },
+ {
+ "sites": [
+ "map",
+ "shopping_admin"
+ ],
+ "task_id": 759,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the route and driving time from {{city1}} to {{city2}}",
+ "instantiation_dict": {
+ "city1": "the city where my E-commerce customer Sophia Young lives",
+ "city2": "New York City"
+ },
+ "intent": "Show me the route and driving time from the city where my E-commerce customer Sophia Young lives to New York City",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Boston"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "New York"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 42
+ },
+ {
+ "sites": [
+ "map",
+ "shopping_admin"
+ ],
+ "task_id": 760,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Show me the route and driving time from {{city1}} to {{city2}}",
+ "instantiation_dict": {
+ "city1": "Allentown, PA",
+ "city2": "the city where my E-commerce customer Amanda Kim lives"
+ },
+ "intent": "Show me the route and driving time from Allentown, PA to the city where my E-commerce customer Amanda Kim lives",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Allentown"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Hoboken",
+ "New Jersey"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 42
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 761,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Get directions from {{location/address_1}} to {{location/address_2}} using {{transportation}} options.",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Science Museum",
+ "location/address_2": "Hunt library CMU",
+ "transportation": "walk"
+ },
+ "intent": "Get directions from Carnegie Science Museum to Hunt library CMU using walk options.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Science Center",
+ "Allegheny County",
+ "Pittsburgh"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Hunt Library",
+ "Pittsburgh"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 54
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 762,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Get directions from {{location/address_1}} to {{location/address_2}} using {{transportation}} options.",
+ "instantiation_dict": {
+ "location/address_1": "Carnegie Music Hall in NYC",
+ "location/address_2": "Carnegie Mellon University",
+ "transportation": "driving"
+ },
+ "intent": "Get directions from Carnegie Music Hall in NYC to Carnegie Mellon University using driving options.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Hall",
+ "West 57th Street",
+ "Manhattan",
+ "New York"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Carnegie Mellon University",
+ "Pittsburgh"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 54
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 763,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
+ "instantiation_dict": {
+ "store": "Trader Joe's",
+ "location": "401 Shady Ave, Pittsburgh"
+ },
+ "intent": "Find the walkway to the closest Trader Joe's from 401 Shady Ave, Pittsburgh.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "401, Shady Avenue, Shadyside"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Trader Joe's, 6343, Penn Avenue, East Liberty"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 75
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 764,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
+ "instantiation_dict": {
+ "store": "Target",
+ "location": "401 Shady Ave, Pittsburgh"
+ },
+ "intent": "Find the walkway to the closest Target from 401 Shady Ave, Pittsburgh.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "401, Shady Avenue, Shadyside"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Target, 6231, Penn Avenue, East Liberty"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 75
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 765,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
+ "instantiation_dict": {
+ "store": "Japanese food market",
+ "location": "401 Shady Ave, Pittsburgh"
+ },
+ "intent": "Find the walkway to the closest Japanese food market from 401 Shady Ave, Pittsburgh.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "401, Shady Avenue, Shadyside"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Tokyo Japanese Food Store, 5855, Ellsworth Avenue, Shadyside"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 75
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 766,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
+ "instantiation_dict": {
+ "store": "grocessory owned by Amazon",
+ "location": "401 Shady Ave, Pittsburgh"
+ },
+ "intent": "Find the walkway to the closest grocessory owned by Amazon from 401 Shady Ave, Pittsburgh.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "401, Shady Avenue, Shadyside"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Whole Foods Market, 5700, Penn Avenue, East Liberty"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 75
+ },
+ {
+ "sites": [
+ "map"
+ ],
+ "task_id": 767,
+ "require_login": true,
+ "storage_state": null,
+ "start_url": "__MAP__",
+ "geolocation": null,
+ "intent_template": "Find the walkway to the closest {{store}} from {{location}}.",
+ "instantiation_dict": {
+ "store": "chain grocessory owned by a local business",
+ "location": "401 Shady Ave, Pittsburgh"
+ },
+ "intent": "Find the walkway to the closest chain grocessory owned by a local business from 401 Shady Ave, Pittsburgh.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector(\"div#content select.routing_engines\").selectedIndex",
+ "required_contents": {
+ "exact_match": "2"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_from\"').value",
+ "required_contents": {
+ "must_include": [
+ "401, Shady Avenue, Shadyside"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[name=\"route_to\"').value",
+ "required_contents": {
+ "must_include": [
+ "Giant Eagle, 5550, Centre Avenue, Shadyside"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 75
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 768,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{quantity}} {{product}} arrived, update the stock",
+ "instantiation_dict": {
+ "quantity": "5",
+ "product": "blue Cronus yoga pants with size 33"
+ },
+ "intent": "5 blue Cronus yoga pants with size 33 arrived, update the stock",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/872/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "5"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/872/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][is_in_stock]\"').value",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 241
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 769,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "We've received {{quantity}} {{product}}, please update the inventory.",
+ "instantiation_dict": {
+ "quantity": "378",
+ "product": "brown Aero daily fitness tee in every size"
+ },
+ "intent": "We've received 378 brown Aero daily fitness tee in every size, please update the inventory.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/544/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "478"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/547/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "478"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/550/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "478"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/553/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "478"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/556/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "478"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 241
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 770,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "We've received {{quantity}}, update the inventory.",
+ "instantiation_dict": {
+ "quantity": "12 white Cora parachute pant of size 28 and 56 blue of size 29"
+ },
+ "intent": "We've received 12 white Cora parachute pant of size 28 and 56 blue of size 29, update the inventory.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1836/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "112"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1838/",
+ "locator": "document.querySelector('[name=\"product[quantity_and_stock_status][qty]\"').value",
+ "required_contents": {
+ "exact_match": "156"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 241
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 771,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Approve the positive reviews to display in our store.",
+ "instantiation_dict": {},
+ "intent": "Approve the positive reviews to display in our store.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/352",
+ "locator": "document.querySelector('[name=\"status_id\"').value",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/349",
+ "locator": "document.querySelector('[name=\"status_id\"').value",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/347",
+ "locator": "document.querySelector('[name=\"status_id\"').value",
+ "required_contents": {
+ "exact_match": "1"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 243
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 772,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "pending negative reviews for Circe fleece"
+ },
+ "intent": "Delete all pending negative reviews for Circe fleece",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/999",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 773,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "pending negative reviews"
+ },
+ "intent": "Delete all pending negative reviews",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/351",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/353",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 774,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "pending reviews with less than 4 stars"
+ },
+ "intent": "Delete all pending reviews with less than 4 stars",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/351",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/353",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/349",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 775,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "reviews from the scammer Arden"
+ },
+ "intent": "Delete all reviews from the scammer Arden",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/51",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 776,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "reviews from the scammer Carlo"
+ },
+ "intent": "Delete all reviews from the scammer Carlo",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/93",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/review/product/edit/id/109",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Rating isn't Available"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 777,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "$5",
+ "action": "Reduce",
+ "config": "green Hollister backyard sweater in all size"
+ },
+ "intent": "Reduce the price of green Hollister backyard sweater in all size by $5",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/120/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "47.00"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/117/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "47.00"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/114/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "47.00"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/111/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "47.00"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/123/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "47.00"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 778,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "13.5%",
+ "action": "Reduce",
+ "config": "size 28 Sahara leggings"
+ },
+ "intent": "Reduce the price of size 28 Sahara leggings by 13.5%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1841/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "64.88"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1842/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "64.88"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1843/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "64.88"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 779,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "15%",
+ "action": "Reduce",
+ "config": "yellow shirts from Gwyn Endurance in all size below L"
+ },
+ "intent": "Reduce the price of yellow shirts from Gwyn Endurance in all size below L by 15%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1559/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "20.40"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1562/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "20.40"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1565/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "20.40"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 780,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1481/",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "$17",
+ "action": "Increase",
+ "config": "white Ingrid Running with size L and above"
+ },
+ "intent": "Increase the price of white Ingrid Running with size L and above by $17",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1264/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "64.00"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1267/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "64.00"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 781,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "37%",
+ "action": "Increase",
+ "config": "black fitness tshirts from Desiree with size XS"
+ },
+ "intent": "Increase the price of black fitness tshirts from Desiree with size XS by 37%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/1573/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "32.88"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 782,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "{{action}} the price of {{config}} by {{amount}}",
+ "instantiation_dict": {
+ "amount": "23%",
+ "action": "Increase",
+ "config": "all blue running tshirts in extra small and small sizes"
+ },
+ "intent": "Increase the price of all blue running tshirts in extra small and small sizes by 23%",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/496/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "22.33"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/499/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "22.33"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/479/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "21.56"
+ }
+ },
+ {
+ "url": "__SHOPPING_ADMIN__/catalog/product/edit/id/482/",
+ "locator": "document.querySelector('[name=\"product[price]\"').value",
+ "required_contents": {
+ "exact_match": "21.56"
+ }
+ }
+ ]
+ },
+ "intent_template_id": 742
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 783,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Add the following users to repo {{repo}} as {{role}}: {{user_list}}",
+ "instantiation_dict": {
+ "repo": "kkroening/ffmpeg-python",
+ "role": "maintainer",
+ "user_list": [
+ "yjlou",
+ "a11yproject"
+ ]
+ },
+ "intent": "Add the following users to repo kkroening/ffmpeg-python as maintainer: ['yjlou', 'a11yproject']",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "not the owner of the repo so cannot add maintainer",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 351
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 784,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/CellularPrivacy/Android-IMSI-Catcher-Detector",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
+ "instantiation_dict": {
+ "branch_name": "main",
+ "attribute": "email address"
+ },
+ "intent": "Tell me the email address of the contributor who has the most commits to branch main",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "secupwn@users.noreply.github.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "secupwn@users.noreply.github.com"
+ },
+ "intent_template_id": 316
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 785,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/CellularPrivacy/Android-IMSI-Catcher-Detector",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
+ "instantiation_dict": {
+ "branch_name": "gh-page",
+ "attribute": "email address"
+ },
+ "intent": "Tell me the email address of the contributor who has the most commits to branch gh-page",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "secupwn@users.noreply.github.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "secupwn@users.noreply.github.com"
+ },
+ "intent_template_id": 316
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 786,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/vinta/awesome-python",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
+ "instantiation_dict": {
+ "branch_name": "main",
+ "attribute": "number of commits"
+ },
+ "intent": "Tell me the number of commits of the contributor who has the most commits to branch main",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "412"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "412"
+ },
+ "intent_template_id": 316
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 787,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/dehenne/awesome-visibility/-/graphs/master",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
+ "instantiation_dict": {
+ "branch_name": "main",
+ "attribute": "number of followers"
+ },
+ "intent": "Tell me the number of followers of the contributor who has the most commits to branch main",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "0"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "0"
+ },
+ "intent_template_id": 316
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 788,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/amwhalen/archive-my-tweets/-/tree/github/fork/chtitux/addRssFeed",
+ "geolocation": null,
+ "intent_template": "Tell me the {{attribute}} of the contributor who has the most commits to branch {{branch_name}}",
+ "instantiation_dict": {
+ "branch_name": "php52",
+ "attribute": "full name, gitlab account name, location and email address"
+ },
+ "intent": "Tell me the full name, gitlab account name, location and email address of the contributor who has the most commits to branch php52",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "must_include": [
+ "Andrew M. Whalen",
+ "amwhalen",
+ "Massachusetts",
+ "github@amwhalen.com"
+ ]
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "",
+ "reference_answer_raw_annotation": "Andrew M. Whalen, amwhalen, Massachusetts, github@amwhalen.com"
+ },
+ "intent_template_id": 316
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 789,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue {{issue}} in {{repo}}.",
+ "instantiation_dict": {
+ "repo": "huggingface dataset",
+ "issue": "asking about do they have any plan on supporting Webagent in the next quater"
+ },
+ "intent": "Create an issue asking about do they have any plan on supporting Webagent in the next quater in huggingface dataset.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "huggingface repo does not exist",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 328
+ },
+ {
+ "sites": [
+ "shopping_admin"
+ ],
+ "task_id": 790,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_admin_state.json",
+ "start_url": "__SHOPPING_ADMIN__",
+ "geolocation": null,
+ "intent_template": "Delete all {{review_type}}",
+ "instantiation_dict": {
+ "review_type": "negative reviews for Sybil running short"
+ },
+ "intent": "Delete all negative reviews for Sybil running short",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "No negative reviews for sybil running short",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 246
+ },
+ {
+ "sites": [
+ "gitlab",
+ "reddit"
+ ],
+ "task_id": 791,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab.reddit_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Make a folder named {{directory}} on the {{gitlab_repo}} repo and include a file called urls.txt that consists of the links to the 5 most recent posts from {{subreddit}}.",
+ "instantiation_dict": {
+ "directory": "car",
+ "subreddit": "cars",
+ "gitlab_repo": "gimmiethat.space"
+ },
+ "intent": "Make a folder named car on the gimmiethat.space repo and include a file called urls.txt that consists of the links to the 5 most recent posts from cars.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "the subreddit does not exist",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 84
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 792,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
+ "instantiation_dict": {
+ "product_category": "Beauty & Personal Care",
+ "dollar_value": "under 20"
+ },
+ "intent": "Buy the highest rated product from the Beauty & Personal Care category within a budget under 20.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The website does not support sorting by rating and there are too many products in this category",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 172
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 793,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Buy the highest rated product from the {{product_category}} category within a budget {{dollar_value}}.",
+ "instantiation_dict": {
+ "product_category": "Men clothing",
+ "dollar_value": "above 50 but under 129.99"
+ },
+ "intent": "Buy the highest rated product from the Men clothing category within a budget above 50 but under 129.99.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The website does not support sorting by rating and there are too many products in this category",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 172
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 794,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Change the delivery address for my most recent order to {{address}}.",
+ "instantiation_dict": {
+ "address": "4000 Forbes Ave, Pittsburgh, PA"
+ },
+ "intent": "Change the delivery address for my most recent order to 4000 Forbes Ave, Pittsburgh, PA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The shipping address cannot be changed after an order is placed in the website.",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 191
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 795,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Change the delivery address for my most recent order to {{address}}.",
+ "instantiation_dict": {
+ "address": "6726 McPherson Blvd, Pittsburgh, PA"
+ },
+ "intent": "Change the delivery address for my most recent order to 6726 McPherson Blvd, Pittsburgh, PA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The shipping address cannot be changed after an order is placed in the website.",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 191
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 796,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Change the delivery address for my most recent order to {{address}}.",
+ "instantiation_dict": {
+ "address": "155 5th Street, San Francisco, CA"
+ },
+ "intent": "Change the delivery address for my most recent order to 155 5th Street, San Francisco, CA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The shipping address cannot be changed after an order is placed in the website.",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 191
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 797,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Change the delivery address for my most recent order to {{address}}.",
+ "instantiation_dict": {
+ "address": "3 Oxford St, Cambridge, MA"
+ },
+ "intent": "Change the delivery address for my most recent order to 3 Oxford St, Cambridge, MA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The shipping address cannot be changed after an order is placed in the website.",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 191
+ },
+ {
+ "sites": [
+ "shopping"
+ ],
+ "task_id": 798,
+ "require_login": true,
+ "storage_state": "./.auth/shopping_state.json",
+ "start_url": "__SHOPPING__",
+ "geolocation": null,
+ "intent_template": "Change the delivery address for my most recent order to {{address}}.",
+ "instantiation_dict": {
+ "address": "77 Massachusetts Ave, Cambridge, MA"
+ },
+ "intent": "Change the delivery address for my most recent order to 77 Massachusetts Ave, Cambridge, MA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "N/A"
+ },
+ "reference_url": "",
+ "program_html": [],
+ "string_note": "The shipping address cannot be changed after an order is placed in the website.",
+ "reference_answer_raw_annotation": "N/A"
+ },
+ "intent_template_id": 191
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 799,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a new group \"{{name}}\" with members {{members}}",
+ "instantiation_dict": {
+ "name": "n-lab",
+ "members": "patou, egpast, westurner, jontutcher"
+ },
+ "intent": "create a new group \"n-lab\" with members patou, egpast, westurner, jontutcher",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/groups/n-lab/-/group_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@patou",
+ "@egpast",
+ "@westurner",
+ "@jontutcher"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 600
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 800,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a new group \"{{name}}\" with members {{members}}",
+ "instantiation_dict": {
+ "name": "x-lab",
+ "members": "JonasVautherin, dilipchandima, dawiss1337, bmyun, DCMJY"
+ },
+ "intent": "create a new group \"x-lab\" with members JonasVautherin, dilipchandima, dawiss1337, bmyun, DCMJY",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/groups/x-lab/-/group_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@JonasVautherin",
+ "@dilipchandima",
+ "@dawiss1337",
+ "@bmyun",
+ "@DCMJY"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 600
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 801,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a new group \"{{name}}\" with members {{members}}",
+ "instantiation_dict": {
+ "name": "crew",
+ "members": "ASWATFZLLC, patrickhlauke, westurner, linkmatrix"
+ },
+ "intent": "create a new group \"crew\" with members ASWATFZLLC, patrickhlauke, westurner, linkmatrix",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/groups/crew/-/group_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@ASWATFZLLC",
+ "@patrickhlauke",
+ "@westurner",
+ "@linkmatrix"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 600
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 802,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a new group \"{{name}}\" with members {{members}}",
+ "instantiation_dict": {
+ "name": "coding_friends",
+ "members": "qhduan, Agnes-U"
+ },
+ "intent": "create a new group \"coding_friends\" with members qhduan, Agnes-U",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/groups/coding_friends/-/group_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@qhduan",
+ "@Agnes-U"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 600
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 803,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "create a new group \"{{name}}\" with members {{members}}",
+ "instantiation_dict": {
+ "name": "webagent",
+ "members": "pandey2000, sayakpaul, sayakpaul"
+ },
+ "intent": "create a new group \"webagent\" with members pandey2000, sayakpaul, sayakpaul",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/groups/webagent/-/group_members",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "@pandey2000",
+ "@sayakpaul"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 600
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 804,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Assign the issue regarding {{issue}} to {{account}}.",
+ "instantiation_dict": {
+ "issue": "flash alerts",
+ "account": "myself and primer"
+ },
+ "intent": "Assign the issue regarding flash alerts to myself and primer.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Add documentation on using Flash alerts in dialog components",
+ "Clarify usage of flash alert"
+ ]
+ }
+ },
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=primer",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Add documentation on using Flash alerts in dialog components",
+ "Clarify usage of flash alert"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 999
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 805,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/a11yproject/a11yproject.com",
+ "geolocation": null,
+ "intent_template": "Submit a merge request for {{source_branch}} to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "the branch that implements the support of template strings",
+ "target_branch": "master",
+ "reviewer": "myself and Roshan"
+ },
+ "intent": "Submit a merge request for the branch that implements the support of template strings to be merged into master branch, assign myself and Roshan as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "feature/replace-gulp"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "main"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Byte Blaze",
+ "Roshan Jossy"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 806,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "a11yproject.com/redesign",
+ "target_branch": "markdown-figure-block",
+ "reviewer": "myself"
+ },
+ "intent": "Submit a merge request for a11yproject.com/redesign branch to be merged into markdown-figure-block branch, assign myself as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "redesign"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "feature/markdown-figure-block"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Byte Blaze"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 807,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__/primer/design",
+ "geolocation": null,
+ "intent_template": "Submit a merge request for {{source_branch}} branch to be merged into {{target_branch}} branch, assign {{reviewer}} as the reviewer",
+ "instantiation_dict": {
+ "source_branch": "build time debug",
+ "target_branch": "main",
+ "reviewer": "myself"
+ },
+ "intent": "Submit a merge request for build time debug branch to be merged into main branch, assign myself as the reviewer",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/primer/design/-/merge_requests",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[1].outerText",
+ "required_contents": {
+ "exact_match": "main"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelectorAll(\".detail-page-description > a.gl-font-monospace\")[0].outerText",
+ "required_contents": {
+ "exact_match": "debug-build-time"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.reviewer').outerText",
+ "required_contents": {
+ "must_include": [
+ "Byte Blaze"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 335
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 808,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
+ "instantiation_dict": {
+ "repo": "cloud-to-butt",
+ "issue": "Let's keep the project alive",
+ "account": "myself",
+ "due": "the end of Q1 2033"
+ },
+ "intent": "Create an issue in cloud-to-butt repo with title \"Let's keep the project alive\". Assign the issue to myself. Set due date to be the end of Q1 2033",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/byteblaze/cloud-to-butt/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
+ "required_contents": {
+ "exact_match": "Let's keep the project alive"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Mar 31, 2033"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.assignee').outerText",
+ "required_contents": {
+ "must_include": [
+ "Byte Blaze"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 327
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 809,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Create an issue in {{repo}} repo with title \"{{issue}}\". Assign the issue to {{account}}. Set due date to be {{due}}",
+ "instantiation_dict": {
+ "repo": "a11yproject",
+ "issue": "404 for many URLs",
+ "account": "myself",
+ "due": "2030-1-3"
+ },
+ "intent": "Create an issue in a11yproject repo with title \"404 for many URLs\". Assign the issue to myself. Set due date to be 2030-1-3",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "url_match",
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "__GITLAB__/a11yproject/a11yproject.com/-/issues",
+ "program_html": [
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-qa-selector=\"title_content\"]').outerText",
+ "required_contents": {
+ "exact_match": "404 for many URLs"
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('[data-testid=\"sidebar-due-date\"').outerText",
+ "required_contents": {
+ "must_include": [
+ "Jan 3, 2030"
+ ]
+ }
+ },
+ {
+ "url": "last",
+ "locator": "document.querySelector('.block.assignee').outerText",
+ "required_contents": {
+ "must_include": [
+ "Byte Blaze"
+ ]
+ }
+ }
+ ],
+ "url_note": "GOLD in PRED"
+ },
+ "intent_template_id": 327
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 810,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Assign the issue regarding {{issue}} in {{repo}} repo to {{account}}.",
+ "instantiation_dict": {
+ "repo": "primer design guide",
+ "issue": "flash alert",
+ "account": "myself"
+ },
+ "intent": "Assign the issue regarding flash alert in primer design guide repo to myself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "Add documentation on using Flash alerts in dialog components",
+ "Clarify usage of flash alert"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 999
+ },
+ {
+ "sites": [
+ "gitlab"
+ ],
+ "task_id": 811,
+ "require_login": true,
+ "storage_state": "./.auth/gitlab_state.json",
+ "start_url": "__GITLAB__",
+ "geolocation": null,
+ "intent_template": "Assign the issue regarding {{issue}} in {{repo}} to {{account}}.",
+ "instantiation_dict": {
+ "repo": "a11yproject",
+ "issue": 404,
+ "account": "myself"
+ },
+ "intent": "Assign the issue regarding 404 in a11yproject to myself.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "program_html"
+ ],
+ "reference_answers": null,
+ "reference_url": "",
+ "program_html": [
+ {
+ "url": "__GITLAB__/dashboard/issues?scope=all&state=opened&assignee_username=byteblaze",
+ "locator": "",
+ "required_contents": {
+ "must_include": [
+ "404s, bad host, timeouts, bad urls for URLs linked from website"
+ ]
+ }
+ }
+ ]
+ },
+ "intent_template_id": 999
+ }
+]
diff --git a/config_files/webvoyager/Allrecipes--10.json b/config_files/webvoyager/Allrecipes--10.json
new file mode 100644
index 0000000..5a45e80
--- /dev/null
+++ b/config_files/webvoyager/Allrecipes--10.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "allrecipes"
+ ],
+ "task_id": "Allrecipes--10",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.allrecipes.com/",
+ "geolocation": null,
+ "intent": "Find The Most Popular Recipes of the 1960s, noting the recipe name, preparation time and total time of the second recipe in this collection.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'Swedish Meatballs I', prep time 25 mins, total time 1 hour 25 mins"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Allrecipes--17.json b/config_files/webvoyager/Allrecipes--17.json
new file mode 100644
index 0000000..b285823
--- /dev/null
+++ b/config_files/webvoyager/Allrecipes--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "allrecipes"
+ ],
+ "task_id": "Allrecipes--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.allrecipes.com/",
+ "geolocation": null,
+ "intent": "Find the Easy Vegetarian Spinach Lasagna recipe on Allrecipes and tell me what the latest review says.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Easy to make and very delicious"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Allrecipes--3.json b/config_files/webvoyager/Allrecipes--3.json
new file mode 100644
index 0000000..239f858
--- /dev/null
+++ b/config_files/webvoyager/Allrecipes--3.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "allrecipes"
+ ],
+ "task_id": "Allrecipes--3",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.allrecipes.com/",
+ "geolocation": null,
+ "intent": "Locate a recipe for vegan chocolate chip cookies with over 60 reviews and a rating of at least 4.5 stars on Allrecipes.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'Vegan Chocolate Chip, Oatmeal, and Nut Cookies', 4.9 star, 67 viewers (> 60)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Allrecipes--40.json b/config_files/webvoyager/Allrecipes--40.json
new file mode 100644
index 0000000..0da43c0
--- /dev/null
+++ b/config_files/webvoyager/Allrecipes--40.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "allrecipes"
+ ],
+ "task_id": "Allrecipes--40",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.allrecipes.com/",
+ "geolocation": null,
+ "intent": "Browse the about us section of Allrecipes for a brief introduction to The Allrecipes Allstars.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "The Allrecipes Allstars: Social media influencers, registered dietitians, grillmasters, and more seasoned home cooks make up our enthusiastic squad of 100+ brand ambassadors. This diverse, food-loving crew spans the U.S. geographically and represents many different cultures, ethnicities, and family makeups. Since 2011, the Allstars have created tens of thousands of original recipes, photos, and reviews plus shared their cooking expertise via flat and video content on our website, social media, plus more marketing channels."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Amazon--16.json b/config_files/webvoyager/Amazon--16.json
new file mode 100644
index 0000000..f5af174
--- /dev/null
+++ b/config_files/webvoyager/Amazon--16.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "amazon"
+ ],
+ "task_id": "Amazon--16",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.amazon.com/",
+ "geolocation": null,
+ "intent": "Find the Return Policy for Mens Rhinestone Skull Graphic Shirt on Amazon. Color: Black, Size: XX-Large. If Free return is avaliable, tell me how to return this item.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "FREE Returns, 1. Go to Your Orders to start the return; 2. Print the return shipping label; 3. Ship it!"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--12.json b/config_files/webvoyager/Apple--12.json
new file mode 100644
index 0000000..c264f58
--- /dev/null
+++ b/config_files/webvoyager/Apple--12.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--12",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "What Apple Repair ways are mentioned on apple website, answer 2 of them.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Any 2 of 'Send your product to Apple', 'Find an Apple Authorized Service Provider', 'Visit a Genius at an Apple Store', 'Independent Repair Providers', 'Self Service Repair'"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--17.json b/config_files/webvoyager/Apple--17.json
new file mode 100644
index 0000000..fe547b7
--- /dev/null
+++ b/config_files/webvoyager/Apple--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "Search Apple for the accessory Smart Folio for iPad and check the closest pickup availability next to zip code 90038.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Apple Tower Theatre"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--18.json b/config_files/webvoyager/Apple--18.json
new file mode 100644
index 0000000..aec6032
--- /dev/null
+++ b/config_files/webvoyager/Apple--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "Check if there are trade-in offers for the latest model of iPhone.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "There are trade-in offers."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--19.json b/config_files/webvoyager/Apple--19.json
new file mode 100644
index 0000000..4260ba7
--- /dev/null
+++ b/config_files/webvoyager/Apple--19.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--19",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "On Apple's website, what is the slogan for the Mac and what is the slogan for the Macbook pro.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "If you can dream it, Mac can do it; Mind-blowing. Head-turning"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--28.json b/config_files/webvoyager/Apple--28.json
new file mode 100644
index 0000000..21bfb6e
--- /dev/null
+++ b/config_files/webvoyager/Apple--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "On the Apple website, find out if the Mac Mini can be configured with a GPU larger than 16-core.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Yes. Mac mini Apple M2 Pro chip, Configurable to: 19-core GPU"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--31.json b/config_files/webvoyager/Apple--31.json
new file mode 100644
index 0000000..e000a93
--- /dev/null
+++ b/config_files/webvoyager/Apple--31.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--31",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "On Apple's website, what is the slogan for the latest Apple Watch Series.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Smarter. Brighter. Mightier."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Apple--37.json b/config_files/webvoyager/Apple--37.json
new file mode 100644
index 0000000..6451dd6
--- /dev/null
+++ b/config_files/webvoyager/Apple--37.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "apple"
+ ],
+ "task_id": "Apple--37",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.apple.com/",
+ "geolocation": null,
+ "intent": "Compare the color options of iPhone 13 Pro, iPhone 14 Pro and iPhone 15 Pro.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "iPhone 13 pro: Alpine Green, Silver, Gold, Graphite, Sierra Blue; iPhone 14 pro: Deep Purple, Gold, Silver, Space Black; iPhone 15 pro: Natural Titanium, Blue Titanium, White Titanium, Black Titanium"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--10.json b/config_files/webvoyager/ArXiv--10.json
new file mode 100644
index 0000000..33f69f8
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--10.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--10",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Visit ArXiv Help on how to withdraw an article if the submission is not yet announced.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "If your submission has not yet become publicly available you may delete or delay it. To do either of these things go to your user page and select either the Delete or Unsubmit icon."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--11.json b/config_files/webvoyager/ArXiv--11.json
new file mode 100644
index 0000000..a254ca4
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--11.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--11",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "For Non-English submissions, do I need to provide a multi-language abstract, if need, answer the separator between the multiple abstracts.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "-----"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--12.json b/config_files/webvoyager/ArXiv--12.json
new file mode 100644
index 0000000..d8eec34
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--12.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--12",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Find store in arXiv Help, tell me how many styles of arXiv Logo Shirt are available?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "3"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--14.json b/config_files/webvoyager/ArXiv--14.json
new file mode 100644
index 0000000..feb95db
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--14.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--14",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "On ArXiv, how many articles have 'SimCSE' in the article and are originally announced in October 2023?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "3"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--17.json b/config_files/webvoyager/ArXiv--17.json
new file mode 100644
index 0000000..35dff74
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Find the paper 'GPT-4 Technical Report', when was v3 submitted?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Mon, 27 Mar 2023 17:46:54 UTC"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--18.json b/config_files/webvoyager/ArXiv--18.json
new file mode 100644
index 0000000..141a945
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Download the paper 'Dense Passage Retrieval for Open-Domain Question Answering'. How many formulas are in the article and which one is the loss function?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "2 formulas, the second one is loss function"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--25.json b/config_files/webvoyager/ArXiv--25.json
new file mode 100644
index 0000000..f22b72f
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--25.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--25",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Browse the ArXiv store and let me know how many different types of merchandise are available.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "6, arXiv Logo Shirt, arXiv Logo Mug, arXiv is Open Science, Gift cards, arXiv Morning Mug, arXiv Forever"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--27.json b/config_files/webvoyager/ArXiv--27.json
new file mode 100644
index 0000000..536422d
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--27.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--27",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "On ArXiv, what categories does Economics include, and what are their abbreviations?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Econometrics (econ.EM), General Economics (econ.GN), and Theoretical Economics (econ.TH)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--30.json b/config_files/webvoyager/ArXiv--30.json
new file mode 100644
index 0000000..77366cf
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--30.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--30",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Look up the submission guidelines on ArXiv for submitting a paper and tell me the formats for figures.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Accepted figure formats: PostScript (PS, EPS) \u2014 requires LaTeX processing; JPEG, GIF, PNG or PDF figures \u2014 requires PDFLaTeX processing"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--31.json b/config_files/webvoyager/ArXiv--31.json
new file mode 100644
index 0000000..401d9b6
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--31.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--31",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Search ArXiv for papers with 'Graph Neural Networks' in the abstract that were submitted between Jan 1, 2024, and Jan 3, 2024, and determine how many of these papers have more than five authors.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "7 papers"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--36.json b/config_files/webvoyager/ArXiv--36.json
new file mode 100644
index 0000000..d62b32f
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--36.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--36",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Search 'CVPR 2023' and 'CVPR2023' through journal ref on ArXiv to see how many results there are respectively.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'CVPR 2023': 48 results; 'CVPR2023': 9 results"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--37.json b/config_files/webvoyager/ArXiv--37.json
new file mode 100644
index 0000000..687cc97
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--37.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--37",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Find the names of people in ArXiv's Leadership Team.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Ramin Zabih, Yoav Artzi, Stephanie Orphan, Steinn Sigurdsson, and Charles Frankston."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--39.json b/config_files/webvoyager/ArXiv--39.json
new file mode 100644
index 0000000..af7de6b
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--39.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--39",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Search the title 'GPT-4 Technical Report' and access this paper through HTML format. Read the paper on this page and tell me what is 'one of the main goals of developing such models' mentioned in the Introduction.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "One of the main goals of developing such models is to improve their ability to understand and generate natural language text, particularly in more complex and nuanced scenarios."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--41.json b/config_files/webvoyager/ArXiv--41.json
new file mode 100644
index 0000000..0e86f05
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--41.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--41",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Find the button to share arxiv non-profit store and follow the QR code to share the shop. Then add arXiv Forever short sleeve (XL) to your cart.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "QR code image, Action: add to chart"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--42.json b/config_files/webvoyager/ArXiv--42.json
new file mode 100644
index 0000000..2d82cd0
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--42.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--42",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "Find an article published between 1 January 2000 and 1 January 2005 that requires Support Vector Machines in the title and its Journey ref is ACL Workshop.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'Using a Support-Vector Machine for Japanese-to-English Translation of Tense, Aspect, and Modality'"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ArXiv--6.json b/config_files/webvoyager/ArXiv--6.json
new file mode 100644
index 0000000..74ae28e
--- /dev/null
+++ b/config_files/webvoyager/ArXiv--6.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "arxiv"
+ ],
+ "task_id": "ArXiv--6",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://arxiv.org/",
+ "geolocation": null,
+ "intent": "How many figures and tables are in the paper \"On the Sentence Embeddings from Pre-trained Language Models\"?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "2 Figures, 8 Tables."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/BBC News--28.json b/config_files/webvoyager/BBC News--28.json
new file mode 100644
index 0000000..2943dc3
--- /dev/null
+++ b/config_files/webvoyager/BBC News--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "bbc news"
+ ],
+ "task_id": "BBC News--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.bbc.com/news/",
+ "geolocation": null,
+ "intent": "Find the Market Data section on BBC News and tell me which company the data comes from.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Business - Market Data, Source: Morningstar"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/BBC News--5.json b/config_files/webvoyager/BBC News--5.json
new file mode 100644
index 0000000..c45f372
--- /dev/null
+++ b/config_files/webvoyager/BBC News--5.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "bbc news"
+ ],
+ "task_id": "BBC News--5",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.bbc.com/news/",
+ "geolocation": null,
+ "intent": "Find the article \"What is climate change? A really simple guide\" and use it to answer what human activities are causing climate change.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "This recent climate change has been caused by human activity, mainly the widespread use of fossil fuels - coal, oil and gas - in homes, factories and transport."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Booking--33.json b/config_files/webvoyager/Booking--33.json
new file mode 100644
index 0000000..1cfad51
--- /dev/null
+++ b/config_files/webvoyager/Booking--33.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "booking"
+ ],
+ "task_id": "Booking--33",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.booking.com/",
+ "geolocation": null,
+ "intent": "Find the Customer Service on the Booking website, browse the questions about cancellation, and tell me 'how do I know whether my booking has been cancelled'.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "After you cancel a booking with us, you should get an email confirming the cancellation. Make sure to check your inbox and spam/junk mail folders. If you don\u2019t receive an email within 24 hours, contact the property to confirm they got your cancellation."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Booking--41.json b/config_files/webvoyager/Booking--41.json
new file mode 100644
index 0000000..da9c2c9
--- /dev/null
+++ b/config_files/webvoyager/Booking--41.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "booking"
+ ],
+ "task_id": "Booking--41",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.booking.com/",
+ "geolocation": null,
+ "intent": "Browse Booking's homepage to find out which company it belongs to.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Booking Holdings Inc."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--0.json b/config_files/webvoyager/Cambridge Dictionary--0.json
new file mode 100644
index 0000000..b47f223
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--0.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--0",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Look up the pronunciation and definition of the word \"sustainability\" on the Cambridge Dictionary.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "UK: /s\u0259\u02ccste\u026a.n\u0259\u02c8b\u026al.\u0259.ti/, US: /s\u0259\u02ccste\u026a.n\u0259\u02c8b\u026al.\u0259.t\u032ci/; the quality of being able to continue over a period of time"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--17.json b/config_files/webvoyager/Cambridge Dictionary--17.json
new file mode 100644
index 0000000..efb956c
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "How many meanings of \"unblemished\" are given in Cambridge Dictionary? Please browse the page and give the number directly.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "2"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--18.json b/config_files/webvoyager/Cambridge Dictionary--18.json
new file mode 100644
index 0000000..5517418
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Search for \"to behave well\" in Cambridge Dictionary's Thesaurus and see which synonyms the dictionary gives.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "behaves themselves; be on their best behaviour"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--19.json b/config_files/webvoyager/Cambridge Dictionary--19.json
new file mode 100644
index 0000000..4957f78
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--19.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--19",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Try a Cambridge Dictionary translation and tell me which company provided the translation.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Microsoft"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--21.json b/config_files/webvoyager/Cambridge Dictionary--21.json
new file mode 100644
index 0000000..c0fa182
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--21.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--21",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Search for the word \"ephemeral\" on Cambridge Dictionary and find its translation into Spanish.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "ef\u00edmero"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--26.json b/config_files/webvoyager/Cambridge Dictionary--26.json
new file mode 100644
index 0000000..c4c535e
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--26.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--26",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Search for the word \"nostalgia\" in the Cambridge Dictionary and report the translation of this word into Chinese.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "\u6000\u65e7"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--28.json b/config_files/webvoyager/Cambridge Dictionary--28.json
new file mode 100644
index 0000000..420fcb4
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Search for \"feel giddy\" in Cambridge Dictionary's Thesaurus and list the synonyms the dictionary provides.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Synonyms: feel dizzy; whirl; spin; reel"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--42.json b/config_files/webvoyager/Cambridge Dictionary--42.json
new file mode 100644
index 0000000..ea8e246
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--42.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--42",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Convert the Cambridge Dictionary homepage from English (UK) to Deutsch.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Action: Click English (UK), change language to: Deutsch"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Cambridge Dictionary--6.json b/config_files/webvoyager/Cambridge Dictionary--6.json
new file mode 100644
index 0000000..acda793
--- /dev/null
+++ b/config_files/webvoyager/Cambridge Dictionary--6.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "cambridge dictionary"
+ ],
+ "task_id": "Cambridge Dictionary--6",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://dictionary.cambridge.org/",
+ "geolocation": null,
+ "intent": "Search for the word \"sustainability\" on the Cambridge Dictionary, what is the translation of sustainability into Chinese and French in the dictionary.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "\u53ef\u6301\u7eed\u6027; durabilit\u00e9 , viabilit\u00e9"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Coursera--38.json b/config_files/webvoyager/Coursera--38.json
new file mode 100644
index 0000000..011fe6d
--- /dev/null
+++ b/config_files/webvoyager/Coursera--38.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "coursera"
+ ],
+ "task_id": "Coursera--38",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.coursera.org/",
+ "geolocation": null,
+ "intent": "Browse Coursera, which universities and companies from Australia are partners of Coursera? List all of them.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Macquarie University; The University of Melbourne; The University of Sydney; University of Western Australia; UNSW Sydney (The University of New South Wales)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Coursera--39.json b/config_files/webvoyager/Coursera--39.json
new file mode 100644
index 0000000..258a25c
--- /dev/null
+++ b/config_files/webvoyager/Coursera--39.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "coursera"
+ ],
+ "task_id": "Coursera--39",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.coursera.org/",
+ "geolocation": null,
+ "intent": "Find the Space Safety course offered by TUM on Coursera. How many videos are there in module 2? What is the name of each video?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "6 videos; Introduction; Space Debris; Mitigation; Measurements; Protection; Atmospheric Re-entry"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--11.json b/config_files/webvoyager/ESPN--11.json
new file mode 100644
index 0000000..8829029
--- /dev/null
+++ b/config_files/webvoyager/ESPN--11.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--11",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "How many NBA teams are there and list all the teams with 'New' in their name.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "30; New York Knicks; New Orleans Pelicans"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--15.json b/config_files/webvoyager/ESPN--15.json
new file mode 100644
index 0000000..2078672
--- /dev/null
+++ b/config_files/webvoyager/ESPN--15.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--15",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Check the scores of the NBA games played on December 25, 2023.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "(US Time) Bucks vs Knicks, 122 - 129; Warriors vs Nuggets, 114 - 120; Celtics vs Lakers, 126 - 115; 76ers vs Heat, 113 - 119; Mavericks vs Suns, 128 - 114"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--17.json b/config_files/webvoyager/ESPN--17.json
new file mode 100644
index 0000000..05a7d1d
--- /dev/null
+++ b/config_files/webvoyager/ESPN--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Check out the NBA Basketball Power Index 2023-24 to see which teams are in first place and which are in last place.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Boston Celtics; San Antonio Spurs"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--18.json b/config_files/webvoyager/ESPN--18.json
new file mode 100644
index 0000000..4170f69
--- /dev/null
+++ b/config_files/webvoyager/ESPN--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "How many sports leagues can you choose from on the ESPN home page?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "31 (in ESPN America)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--19.json b/config_files/webvoyager/ESPN--19.json
new file mode 100644
index 0000000..d9f961f
--- /dev/null
+++ b/config_files/webvoyager/ESPN--19.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--19",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Who has the highest salary in Boston Celtics Roster 2023-24?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Jrue Holiday"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--27.json b/config_files/webvoyager/ESPN--27.json
new file mode 100644
index 0000000..70e7ee9
--- /dev/null
+++ b/config_files/webvoyager/ESPN--27.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--27",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Search on ESPN for how many teams have 'Golden' in their name and how many of them are in the NHL.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "30 teams in search results, 1 team Vegas Golden Knights (NHL)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--28.json b/config_files/webvoyager/ESPN--28.json
new file mode 100644
index 0000000..717cc42
--- /dev/null
+++ b/config_files/webvoyager/ESPN--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "How many MLB teams are there and list all the teams with 'City' in their name.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "30 teams in search results, Kansas City Royals"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--31.json b/config_files/webvoyager/ESPN--31.json
new file mode 100644
index 0000000..a32b282
--- /dev/null
+++ b/config_files/webvoyager/ESPN--31.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--31",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Who has the heaviest weight among infielders in the New York Yankees Roster 2023-24?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Carlos Rodon, 255 lbs"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--41.json b/config_files/webvoyager/ESPN--41.json
new file mode 100644
index 0000000..6f5686e
--- /dev/null
+++ b/config_files/webvoyager/ESPN--41.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--41",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Find out which four teams the NFC North contains in the NFL on ESPN.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Chicago Bears, Detroit Lions, Green Bay Packers, and Minnesota Vikings"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/ESPN--9.json b/config_files/webvoyager/ESPN--9.json
new file mode 100644
index 0000000..6139efd
--- /dev/null
+++ b/config_files/webvoyager/ESPN--9.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "espn"
+ ],
+ "task_id": "ESPN--9",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.espn.com/",
+ "geolocation": null,
+ "intent": "Search on ESPN for how many teams have Los Angeles in their name and how many of them are NBA.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "10 teams have Los Angeles in their name; 2 teams are NBA"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--1.json b/config_files/webvoyager/Google Map--1.json
new file mode 100644
index 0000000..c0f1849
--- /dev/null
+++ b/config_files/webvoyager/Google Map--1.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--1",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Tell me one bus stop that is nearest to the intersection of main street and Amherst street in Altavista.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'Amherst and 7th' or 'Main Street Middle'"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--21.json b/config_files/webvoyager/Google Map--21.json
new file mode 100644
index 0000000..690d8cd
--- /dev/null
+++ b/config_files/webvoyager/Google Map--21.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--21",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Identify the nearest bus stop to the corner of Elm Street and Oak Street in Massachusetts.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Elm Street & Oak Street, 18 Bay St, Amesbury, MA 01913"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--26.json b/config_files/webvoyager/Google Map--26.json
new file mode 100644
index 0000000..ab2a15c
--- /dev/null
+++ b/config_files/webvoyager/Google Map--26.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--26",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Search for Los Angeles on Google Map, try to print the map as PDF and summarize the information on the map.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": ", print PDF"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--28.json b/config_files/webvoyager/Google Map--28.json
new file mode 100644
index 0000000..3a8546c
--- /dev/null
+++ b/config_files/webvoyager/Google Map--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Find the search settings for Google Map, what options are shown on that page?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Privacy & Safety: Activity, Content, More options; Other settings"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--31.json b/config_files/webvoyager/Google Map--31.json
new file mode 100644
index 0000000..077bb55
--- /dev/null
+++ b/config_files/webvoyager/Google Map--31.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--31",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "First search New York's Central Park Zoo on Google Map, and then find the way to share the map. What is the generated sharing link?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "share link, https://maps.app.goo.gl/Bnp4Q67dTHoFZ4Lx8"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--33.json b/config_files/webvoyager/Google Map--33.json
new file mode 100644
index 0000000..b2a6f43
--- /dev/null
+++ b/config_files/webvoyager/Google Map--33.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--33",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Check out Denver International Airport's information and tell me: 1) which level has the least proportion in reviews; 2) what are its Accessibility and Amenities.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "star 2 has the least proportion; Accessibility: Assistive hearing loop; Wheelchair accessible entrance; Wheelchair accessible parking lot; Wheelchair accessible restroom; Wheelchair accessible seating; Amenities: Baggage storage; Wi-Fi; Free Wi-Fi"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--7.json b/config_files/webvoyager/Google Map--7.json
new file mode 100644
index 0000000..8f97c65
--- /dev/null
+++ b/config_files/webvoyager/Google Map--7.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--7",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Find bus stops in Alanson, MI",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Alanson, MI (EZ-Mart) Bus Stop"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--8.json b/config_files/webvoyager/Google Map--8.json
new file mode 100644
index 0000000..0a9e746
--- /dev/null
+++ b/config_files/webvoyager/Google Map--8.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--8",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Find a place to climb within 2 miles of zip code 90028.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Hollywood Boulders"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Map--9.json b/config_files/webvoyager/Google Map--9.json
new file mode 100644
index 0000000..89ffec7
--- /dev/null
+++ b/config_files/webvoyager/Google Map--9.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google map"
+ ],
+ "task_id": "Google Map--9",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/maps/",
+ "geolocation": null,
+ "intent": "Find the art gallery that is nearest to Los Angeles Hindu Temple.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "'Honor Fraser Gallery' or 'Walter Maciel Gallery'."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--0.json b/config_files/webvoyager/Google Search--0.json
new file mode 100644
index 0000000..dbe8dbc
--- /dev/null
+++ b/config_files/webvoyager/Google Search--0.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--0",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find the initial release date for Guardians of the Galaxy Vol. 3 the movie.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "May 5, 2023"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--12.json b/config_files/webvoyager/Google Search--12.json
new file mode 100644
index 0000000..7a90515
--- /dev/null
+++ b/config_files/webvoyager/Google Search--12.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--12",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find the year that Tom Brady had the most touchdowns in a single seasson.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "2007"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--14.json b/config_files/webvoyager/Google Search--14.json
new file mode 100644
index 0000000..5c0da8c
--- /dev/null
+++ b/config_files/webvoyager/Google Search--14.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--14",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find the retired players the year before last named James Smith and tell me which club he has been a member of from 2020\u20132021.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Yeovil Town"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--15.json b/config_files/webvoyager/Google Search--15.json
new file mode 100644
index 0000000..40917e6
--- /dev/null
+++ b/config_files/webvoyager/Google Search--15.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--15",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Please try to log in to twitter with email: webagenttest@testmail.com and password: test123456. Let me know if the login was successful.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Not successful"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--17.json b/config_files/webvoyager/Google Search--17.json
new file mode 100644
index 0000000..513cfc2
--- /dev/null
+++ b/config_files/webvoyager/Google Search--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Tell me the names of Trump's kids",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Ivanka Trump, Barron Trump, Donald Trump Jr., Tiffany Trump, Eric Trump"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--18.json b/config_files/webvoyager/Google Search--18.json
new file mode 100644
index 0000000..ad95b8c
--- /dev/null
+++ b/config_files/webvoyager/Google Search--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "When and where the most recent World Cup was held, and which team was the winner?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Qatar; November 20 to December 18, 2022; Argentina"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--19.json b/config_files/webvoyager/Google Search--19.json
new file mode 100644
index 0000000..9b7f051
--- /dev/null
+++ b/config_files/webvoyager/Google Search--19.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--19",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "What are the first 7 bits of the SHA of the Bert's latest commit on GitHub, and what exactly was changed in that commit.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "eedf571, Smaller BERT Models"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--20.json b/config_files/webvoyager/Google Search--20.json
new file mode 100644
index 0000000..3140dc1
--- /dev/null
+++ b/config_files/webvoyager/Google Search--20.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--20",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find the release date for the latest \"Fast & Furious\" movie.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "April 4, 2025"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--21.json b/config_files/webvoyager/Google Search--21.json
new file mode 100644
index 0000000..fcc7bca
--- /dev/null
+++ b/config_files/webvoyager/Google Search--21.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--21",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Show a list of the top 5 highest-grossing animated movies, sorted by box office earnings.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "The Lion King (2019); Frozen II (2019); The Super Mario Bros. Movie (2023); Frozen (2013); Incredibles 2 (2018)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--24.json b/config_files/webvoyager/Google Search--24.json
new file mode 100644
index 0000000..30fd53e
--- /dev/null
+++ b/config_files/webvoyager/Google Search--24.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--24",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "What is the name of the star system closest to the Solar System, and what are the discovered planets in it?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Alpha Centauri star system; Proxima Centauri b, Proxima Centauri c, and Proxima Centauri d"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--28.json b/config_files/webvoyager/Google Search--28.json
new file mode 100644
index 0000000..dc0f1c4
--- /dev/null
+++ b/config_files/webvoyager/Google Search--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Check the IMDb and Metacritic scores of the movie \"Inception.\"",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "IMDb score 8.8, Metacritic score 74%."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--29.json b/config_files/webvoyager/Google Search--29.json
new file mode 100644
index 0000000..7420b32
--- /dev/null
+++ b/config_files/webvoyager/Google Search--29.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--29",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find out the current world record for the men's 100m sprint.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "9.58s held by Usain Bolt of Jamaica"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--3.json b/config_files/webvoyager/Google Search--3.json
new file mode 100644
index 0000000..bec9c97
--- /dev/null
+++ b/config_files/webvoyager/Google Search--3.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--3",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Show me a list of comedy movies, sorted by user ratings. Show me the Top 5 movies.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Life Is Beautiful, Back to the Future, The Intouchables, City Lights, Modern Times"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--31.json b/config_files/webvoyager/Google Search--31.json
new file mode 100644
index 0000000..1888ae4
--- /dev/null
+++ b/config_files/webvoyager/Google Search--31.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--31",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Discover which year Cristiano Ronaldo scored the most goals in a single season.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "2014-15 season"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--40.json b/config_files/webvoyager/Google Search--40.json
new file mode 100644
index 0000000..13eed28
--- /dev/null
+++ b/config_files/webvoyager/Google Search--40.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--40",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Look up the elevation of Mount Kilimanjaro on Google Search.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "19,341 feet (5,895 meters)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Google Search--7.json b/config_files/webvoyager/Google Search--7.json
new file mode 100644
index 0000000..c12e8c3
--- /dev/null
+++ b/config_files/webvoyager/Google Search--7.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "google search"
+ ],
+ "task_id": "Google Search--7",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.google.com/",
+ "geolocation": null,
+ "intent": "Find the software requirements for iPhones that support AirDrop's ability to continue transmitting over the web when out of range.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "IOS 17.1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--10.json b/config_files/webvoyager/Huggingface--10.json
new file mode 100644
index 0000000..b87f307
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--10.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--10",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Open space: argilla/notux-chat-ui and interact with it by asking it 'which team trained you'. What is its answer.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Mistral AI team"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--13.json b/config_files/webvoyager/Huggingface--13.json
new file mode 100644
index 0000000..cd9e46e
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--13.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--13",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Search for LLaMA in the huggingface doc, what type is the spaces_between_special_tokens parameter in LlamaTokenizer and what is its default value.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "bool, defaults to False"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--14.json b/config_files/webvoyager/Huggingface--14.json
new file mode 100644
index 0000000..1e04cd0
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--14.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--14",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "How much is the Pro account of Hugging face for a month and what are the features?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "$9/month; Pro Account: Get a PRO badge on your profile, Early access to new features, Unlock Inference for PROs, Higher tier for AutoTrain"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--18.json b/config_files/webvoyager/Huggingface--18.json
new file mode 100644
index 0000000..41ff700
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--18.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--18",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Look up TRL's forward modelling in the hugging face documentation on how to add a margin to a loss.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "As in the Llama 2 paper, you can add a margin to the loss by adding a margin column to the dataset. The reward collator will automatically pass it through and the loss will be computed accordingly."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--20.json b/config_files/webvoyager/Huggingface--20.json
new file mode 100644
index 0000000..b5746af
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--20.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--20",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Locate a pre-trained natural language processing model on Hugging Face that specializes in named entity recognition (NER), confirm that the model was last updated in 2022 and has 1M+ downloads.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "ckiplab/bert-base-chinese-ner"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--21.json b/config_files/webvoyager/Huggingface--21.json
new file mode 100644
index 0000000..76a184f
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--21.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--21",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Look up the tour about how to use the 'pipeline' feature in the Hugging Face Transformers library for sentiment analysis, and identify the default model it uses.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "from transformers import pipeline \\n classifier = pipeline(\"sentiment-analysis\") \\n classifier(\"We are very happy to show you the \ud83e\udd17 Transformers library.\") ... distilbert/distilbert-base-uncased-finetuned-sst-2-english"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--24.json b/config_files/webvoyager/Huggingface--24.json
new file mode 100644
index 0000000..2725db2
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--24.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--24",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Search for a model on Hugging Face with an Apache-2.0 license that has received the highest number of likes.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "mistralai/Mixtral-8x7B-Instruct-v0.1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--25.json b/config_files/webvoyager/Huggingface--25.json
new file mode 100644
index 0000000..3cfc13c
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--25.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--25",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "In the Hugging Face documentation, find the tutorial on loading adapters with PEFT, tell me how to load in 8bit or 4bit.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Add the load_in_8bit or load_in_4bit parameters to from_pretrained() and set device_map=\"auto\" to effectively distribute the model to your hardware. (Or use code)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--27.json b/config_files/webvoyager/Huggingface--27.json
new file mode 100644
index 0000000..4f49e22
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--27.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--27",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Determine the most downloaded dataset related to Text Retrieval in NLP on Hugging Face.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "nlphuji/mscoco_2014_5k_test_image_text_retrieval"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--30.json b/config_files/webvoyager/Huggingface--30.json
new file mode 100644
index 0000000..936314f
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--30.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--30",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Identify the most downloaded English-Chinese (en-zh) machine translation model on Huggingface and report its latest performance metrics and usage guidelines.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Helsinki-NLP/opus-mt-en-zh; testset, BLEU, chr-F: Tatoeba-test.eng.zho, 31.4, 0.268"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--32.json b/config_files/webvoyager/Huggingface--32.json
new file mode 100644
index 0000000..decbe9c
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--32.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--32",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "On the Hugging Face website, search for the model 'GPT-J-6B' and find the 'temperature' parameter in its settings. What is the default value of this parameter?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "\"temperature\": 1.0"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--38.json b/config_files/webvoyager/Huggingface--38.json
new file mode 100644
index 0000000..c977849
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--38.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--38",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Investigate the 'transformers' library in the Hugging Face documentation, focusing on how to add new tokens to a tokenizer.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "use add_tokens method"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--41.json b/config_files/webvoyager/Huggingface--41.json
new file mode 100644
index 0000000..b70eddc
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--41.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--41",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "What is the current Text-to-3D model with the highest number of downloads and tell me are there Spaces that use the model.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "openai/shap-e; there are Spaces like hysts/Shap-E ..."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--42.json b/config_files/webvoyager/Huggingface--42.json
new file mode 100644
index 0000000..26d93e9
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--42.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--42",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Check the Dataset Viewer for ai2lumos/lumos_complex_qa_plan_onetime on Hugging face. what is the content corresponding to user in the first message?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "content: Please provide a reasonable subgoal-based plan to solve the given task.\\nTask: What was the opening date of the museum dedicated to the war that, after it occurred, Boston became one of the wealthiest international ports?; Initial Environment Description: None."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--6.json b/config_files/webvoyager/Huggingface--6.json
new file mode 100644
index 0000000..7f0cde9
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--6.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--6",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Find the model sentence-transformers/all-MiniLM-L6-v2 and use the Inference API on the webpage to get the similarity of the following two sentences: 'Tomorrow is Sunday', 'Eat a burger on Sunday'.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "0.550"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--7.json b/config_files/webvoyager/Huggingface--7.json
new file mode 100644
index 0000000..2bcb721
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--7.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--7",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Which is the most downloaded audio related dataset on Hugging face currently.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "autumnjohnson/ceti_audio"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Huggingface--9.json b/config_files/webvoyager/Huggingface--9.json
new file mode 100644
index 0000000..124fea2
--- /dev/null
+++ b/config_files/webvoyager/Huggingface--9.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "huggingface"
+ ],
+ "task_id": "Huggingface--9",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://huggingface.co/",
+ "geolocation": null,
+ "intent": "Find the most download machine translation model on Huggingface which focuses on English and Japanese (en-ja) and report the evaluation metrics stated for it.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Helsinki-NLP/opus-mt-ja-en; BLEU 41.7\t; chr-F 0.589"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--0.json b/config_files/webvoyager/Wolfram Alpha--0.json
new file mode 100644
index 0000000..cb686ed
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--0.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--0",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "derivative of x^2 when x=5.6",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "11.2"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--10.json b/config_files/webvoyager/Wolfram Alpha--10.json
new file mode 100644
index 0000000..cf66c84
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--10.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--10",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Give the geomagnetic field on June 20, 2023 in Oslo.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "geomagnetic field, total 51.5 uT;"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--11.json b/config_files/webvoyager/Wolfram Alpha--11.json
new file mode 100644
index 0000000..8d15b24
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--11.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--11",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Show the electrical resistivity of UNS A92024 and UNS G10800 at 20 degrees Celsius.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "UNS A92024: 4.9\u00d710^-6 \u03a9 cm (ohm centimeters) (at 20 \u00b0C); UNS G10800: 1.8\u00d710^-5 \u03a9 cm (ohm centimeters)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--12.json b/config_files/webvoyager/Wolfram Alpha--12.json
new file mode 100644
index 0000000..4a5d642
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--12.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--12",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Which character in unicode 8900 to 8920 looks like a snowflake",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "8902 (U+22C6)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--15.json b/config_files/webvoyager/Wolfram Alpha--15.json
new file mode 100644
index 0000000..ddd3a6f
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--15.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--15",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Show the blood relationship fraction between you and your father's mother's sister's son.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "3.125%"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--17.json b/config_files/webvoyager/Wolfram Alpha--17.json
new file mode 100644
index 0000000..c984595
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--17.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--17",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Show the average price of movie ticket in Providence, Nashville, Boise in 2023.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Providence $13.81; Nashville $12.65; Boise $12.65"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--2.json b/config_files/webvoyager/Wolfram Alpha--2.json
new file mode 100644
index 0000000..fb94bf3
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--2.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--2",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate 3^71 and retain 5 significant figures in scientific notation.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "7.5095 * 10^33"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--20.json b/config_files/webvoyager/Wolfram Alpha--20.json
new file mode 100644
index 0000000..8992334
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--20.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--20",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Compute the integral of 3e^(2x) from x=0 to x=5.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "approximately 33038"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--21.json b/config_files/webvoyager/Wolfram Alpha--21.json
new file mode 100644
index 0000000..29d6028
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--21.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--21",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate (1+0.1*i)^8 + (1\u22120.2*i)^8 where i is a complex number.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "approximately 0.717183 - 0.425258 i"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--22.json b/config_files/webvoyager/Wolfram Alpha--22.json
new file mode 100644
index 0000000..f480aa6
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--22.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--22",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Determine the area of a regular hexagon with a side length of 7 cm.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "127.306 cm^2 or 147 \\sqrt(3) / 2"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--23.json b/config_files/webvoyager/Wolfram Alpha--23.json
new file mode 100644
index 0000000..ddd97f0
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--23.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--23",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate the population growth rate of Canada from 2020 to 2023 using Wolfram Alpha.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "mean population growth rate of Canada from 2020 to 2023 is 0.9998% per year"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--24.json b/config_files/webvoyager/Wolfram Alpha--24.json
new file mode 100644
index 0000000..68e67e0
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--24.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--24",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Solve the differential equation y''(t) - 2y'(t) + 10y(t) = 0 and display its general solution.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "y(t) = c1 e^t sin(3t) + c2 e^t cos(3t)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--25.json b/config_files/webvoyager/Wolfram Alpha--25.json
new file mode 100644
index 0000000..e3c5e9d
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--25.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--25",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate the final position and velocity of a projectile launched at 45 degrees with an initial speed of 30 m/s after 3 seconds.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "if g=9.81; x = 63.64m, y = 19.49m; Vx = 21.21 m/s, Vy = -8.22 m/s"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--27.json b/config_files/webvoyager/Wolfram Alpha--27.json
new file mode 100644
index 0000000..d87f201
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--27.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--27",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Display the thermal conductivity of Copper (Cu) and Aluminum (Al) at 25 degrees Celsius.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "401.2 W/(m K); 236.9 W/(m K)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--28.json b/config_files/webvoyager/Wolfram Alpha--28.json
new file mode 100644
index 0000000..1527760
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--28.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--28",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Identify the character in Unicode range 9632 to 9650 that represents a hollow parallelogram.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "9649 or U+25B1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--3.json b/config_files/webvoyager/Wolfram Alpha--3.json
new file mode 100644
index 0000000..5eb276e
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--3.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--3",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Let g(x) be the integral of x^2 cos(2x). Write the expression of g(x).",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "1/4 (2 x cos(2 x) + (-1 + 2 x^2) sin(2 x)) + Constant"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--32.json b/config_files/webvoyager/Wolfram Alpha--32.json
new file mode 100644
index 0000000..f43362b
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--32.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--32",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Print all prime numbers between 1000 and 1200 using Wolfram alpha.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "1009, 1013, 1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069, 1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151, 1153, 1163, 1171, 1181, 1187, 1193."
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--33.json b/config_files/webvoyager/Wolfram Alpha--33.json
new file mode 100644
index 0000000..fa9b90d
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--33.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--33",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Identify the electrical energy output of a hydroelectric power plant named Itaipu Dam in 2023 using Wolfram Alpha.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "89.5 TWh (terawatt hours)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--34.json b/config_files/webvoyager/Wolfram Alpha--34.json
new file mode 100644
index 0000000..9c428b9
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--34.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--34",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate the mass of Jupiter compared to Earth using Wolfram Alpha. Also, find the length of one day on Jupiter.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "317.8 times that of Earth, and the length of one day on Jupiter is approximately 9.925 hours"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--35.json b/config_files/webvoyager/Wolfram Alpha--35.json
new file mode 100644
index 0000000..f515fcb
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--35.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--35",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Calculate the determinant of a 6x6 Hilbert matrix.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "1/186313420339200000"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--36.json b/config_files/webvoyager/Wolfram Alpha--36.json
new file mode 100644
index 0000000..1b1c4b7
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--36.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--36",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Determine the convergence or divergence of the series \u03a3 (n=1 to \u221e) of 1/(n^3 + 1).",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "converges"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--37.json b/config_files/webvoyager/Wolfram Alpha--37.json
new file mode 100644
index 0000000..936a6fd
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--37.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--37",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "How many days are there between February 12, 2024 and August 9, 2050?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "9675"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--38.json b/config_files/webvoyager/Wolfram Alpha--38.json
new file mode 100644
index 0000000..f8adeda
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--38.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--38",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Compute the length of a curve defined by y = 2x^3 - 3x^2 + 4x - 5 from x = 0 to x = 3.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "around 39.2"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--39.json b/config_files/webvoyager/Wolfram Alpha--39.json
new file mode 100644
index 0000000..5876cb3
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--39.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--39",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Use Wolfram alpha to write the expression of the ellipse x^2 + 3 y^2 = 4 rotated 33 degrees counterclockwise.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "x^2(\\sin(\frac{2\u03c0}{15}) - 2) + 2xy \\cos(\frac{2\u03c0}{15}) + 4 = y^2(2 + \\sin(\frac{2\u03c0}{15}))"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--4.json b/config_files/webvoyager/Wolfram Alpha--4.json
new file mode 100644
index 0000000..ff86e05
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--4.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--4",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Pack 24 circles in a circle radius r. Compare Densest known packing and Square packing. Then tell me the radius of the inner circles.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "Densest known packing: 0.176939r; Square packing: 0.163961r"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--40.json b/config_files/webvoyager/Wolfram Alpha--40.json
new file mode 100644
index 0000000..50bcec9
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--40.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--40",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Approximate amount of fat burned by a 28yo, 172cm tall, 70kg woman running for 30min at a pace of 6min/mile.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "around 0.078 kg"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--41.json b/config_files/webvoyager/Wolfram Alpha--41.json
new file mode 100644
index 0000000..2928443
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--41.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--41",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "What is the approximate Heart Rate Reserve of a 50 year old man who has a heart rate of 60bpm at rest.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "110 bpm"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--42.json b/config_files/webvoyager/Wolfram Alpha--42.json
new file mode 100644
index 0000000..6b6e653
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--42.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--42",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "What is the raw memory of a 100.2\" * 123.5\" true colour picture at 72 ppi?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "192 MB"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--43.json b/config_files/webvoyager/Wolfram Alpha--43.json
new file mode 100644
index 0000000..3442437
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--43.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--43",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "A polyominoes of order 6 means you have 6 identical squares to combine different shapes (2-sided). How many combinations are there? Looking at all the shapes in the result, how many of them have only 2 rows in total?",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "35; 12"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--45.json b/config_files/webvoyager/Wolfram Alpha--45.json
new file mode 100644
index 0000000..afafdc5
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--45.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--45",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "A 175cm tall, 85kg, 40yo man climbs 2500 steps at about 18cm per step and 40 steps per minute. summarise the Metabolic properties.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "energy expenditure | 2720 kJ (kilojoules); average energy expenditure per step | 1.1 kJ/step (kilojoules per step); fat burned | 0.0842 kg (kilograms); oxygen consumption | 129.9 L (liters); metabolic equivalent | 7 metabolic equivalents"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--5.json b/config_files/webvoyager/Wolfram Alpha--5.json
new file mode 100644
index 0000000..88c92c7
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--5.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--5",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Show the solution of y\"(z) + sin(y(z)) = 0 from wolframalpha.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "y(z) = \u00b1 2 am(1/2 sqrt((c_1 + 2) (z + c_2)^2), 4/(c_1 + 2)), am(x, m) is the Jacobi amplitude function"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--6.json b/config_files/webvoyager/Wolfram Alpha--6.json
new file mode 100644
index 0000000..d34aa4a
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--6.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--6",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Simplify x^5-20x^4+163x^3-676x^2+1424x-1209 so that it has fewer items.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "7 + 3 (-4 + x)^3 + (-4 + x)^5"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--7.json b/config_files/webvoyager/Wolfram Alpha--7.json
new file mode 100644
index 0000000..b2ccc40
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--7.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--7",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Give the final angle and final length after 6s of a Spring pendulum with spring equilibrium length=0.12m, initial length=0.24m, initial angle=80deg, mass=1kg, spring constant=120 N/m .",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "-73.26\u00b0 from vertical; 0.252 m"
+ }
+ }
+}
\ No newline at end of file
diff --git a/config_files/webvoyager/Wolfram Alpha--9.json b/config_files/webvoyager/Wolfram Alpha--9.json
new file mode 100644
index 0000000..c32cb9e
--- /dev/null
+++ b/config_files/webvoyager/Wolfram Alpha--9.json
@@ -0,0 +1,20 @@
+{
+ "sites": [
+ "wolfram alpha"
+ ],
+ "task_id": "Wolfram Alpha--9",
+ "require_login": false,
+ "storage_state": null,
+ "start_url": "https://www.wolframalpha.com/",
+ "geolocation": null,
+ "intent": "Annual energy production of Diablo Canyon 2 in 2010.",
+ "require_reset": false,
+ "eval": {
+ "eval_types": [
+ "string_match"
+ ],
+ "reference_answers": {
+ "fuzzy_match": "9752 GW h/yr (gigawatt hours per year)"
+ }
+ }
+}
\ No newline at end of file
diff --git a/eval_webarena.py b/eval_webarena.py
new file mode 100644
index 0000000..67051e5
--- /dev/null
+++ b/eval_webarena.py
@@ -0,0 +1,141 @@
+import os
+import time
+import re
+import argparse
+import os
+import shutil
+
+from AgentOccam.env import WebArenaEnvironmentWrapper
+
+from AgentOccam.AgentOccam import AgentOccam
+from webagents_step.utils.data_prep import *
+from webagents_step.agents.step_agent import StepAgent
+
+from AgentOccam.prompts import AgentOccam_prompt
+from webagents_step.prompts.webarena import step_fewshot_template_adapted, step_fewshot_template
+
+from AgentOccam.utils import EVALUATOR_DIR
+
+def run():
+ parser = argparse.ArgumentParser(
+ description="Only the config file argument should be passed"
+ )
+ parser.add_argument(
+ "--config", type=str, required=True, help="yaml config file location"
+ )
+ args = parser.parse_args()
+ with open(args.config, "r") as file:
+ config = DotDict(yaml.safe_load(file))
+
+ if config.logging:
+ if config.logname:
+ dstdir = f"{config.logdir}/{config.logname}"
+ else:
+ dstdir = f"{config.logdir}/{time.strftime('%Y%m%d-%H%M%S')}"
+ os.makedirs(dstdir, exist_ok=True)
+ shutil.copyfile(args.config, os.path.join(dstdir, args.config.split("/")[-1]))
+ random.seed(42)
+
+ config_file_list = []
+
+ task_ids = config.env.task_ids
+ if hasattr(config.env, "relative_task_dir"):
+ relative_task_dir = config.env.relative_task_dir
+ else:
+ relative_task_dir = "tasks"
+ if task_ids == "all" or task_ids == ["all"]:
+ task_ids = [filename[:-len(".json")] for filename in os.listdir(f"config_files/{relative_task_dir}") if filename.endswith(".json")]
+ for task_id in task_ids:
+ config_file_list.append(f"config_files/{relative_task_dir}/{task_id}.json")
+
+ fullpage = config.env.fullpage if hasattr(config.env, "fullpage") else True
+ current_viewport_only = not fullpage
+
+ if config.agent.type == "AgentOccam":
+ agent_init = lambda: AgentOccam(
+ prompt_dict = {k: v for k, v in AgentOccam_prompt.__dict__.items() if isinstance(v, dict)},
+ config = config.agent,
+ )
+ elif config.agent.type == "AgentOccam-SteP":
+ agent_init = lambda: StepAgent(
+ root_action = config.agent.root_action,
+ action_to_prompt_dict = {k: v for k, v in step_fewshot_template_adapted.__dict__.items() if isinstance(v, dict)},
+ low_level_action_list = config.agent.low_level_action_list,
+ max_actions=config.env.max_env_steps,
+ verbose=config.verbose,
+ logging=config.logging,
+ debug=config.debug,
+ model=config.agent.model_name,
+ prompt_mode=config.agent.prompt_mode,
+ )
+ elif config.agent.type == "SteP-replication":
+ agent_init = lambda: StepAgent(
+ root_action = config.agent.root_action,
+ action_to_prompt_dict = {k: v for k, v in step_fewshot_template.__dict__.items() if isinstance(v, dict)},
+ low_level_action_list = config.agent.low_level_action_list,
+ max_actions=config.env.max_env_steps,
+ verbose=config.verbose,
+ logging=config.logging,
+ debug=config.debug,
+ model=config.agent.model_name,
+ prompt_mode=config.agent.prompt_mode,
+ )
+ else:
+ raise NotImplementedError(f"{config.agent.type} not implemented")
+
+
+ for config_file in config_file_list:
+ with open(config_file, "r") as f:
+ task_config = json.load(f)
+ print(f"Task {task_config['task_id']}.")
+ if os.path.exists(os.path.join(dstdir, f"{task_config['task_id']}.json")):
+ print(f"Skip {task_config['task_id']}.")
+ continue
+ if task_config['task_id'] in list(range(600, 650))+list(range(681, 689)):
+ print("Reddit post task. Sleep 30 mins.")
+ time.sleep(1800)
+ env = WebArenaEnvironmentWrapper(config_file=config_file,
+ max_browser_rows=config.env.max_browser_rows,
+ max_steps=config.max_steps,
+ slow_mo=1,
+ observation_type="accessibility_tree",
+ current_viewport_only=current_viewport_only,
+ viewport_size={"width": 1920, "height": 1080},
+ headless=config.env.headless,
+ global_config=config)
+
+ agent = agent_init()
+ objective = env.get_objective()
+ status = agent.act(objective=objective, env=env)
+ env.close()
+
+ if config.logging:
+ with open(config_file, "r") as f:
+ task_config = json.load(f)
+ log_file = os.path.join(dstdir, f"{task_config['task_id']}.json")
+ log_data = {
+ "task": config_file,
+ "id": task_config['task_id'],
+ "model": config.agent.actor.model if hasattr(config.agent, "actor") else config.agent.model_name,
+ "type": config.agent.type,
+ "trajectory": agent.get_trajectory(),
+ }
+ summary_file = os.path.join(dstdir, "summary.csv")
+ summary_data = {
+ "task": config_file,
+ "task_id": task_config['task_id'],
+ "model": config.agent.actor.model if hasattr(config.agent, "actor") else config.agent.model_name,
+ "type": config.agent.type,
+ "logfile": re.search(r"/([^/]+/[^/]+\.json)$", log_file).group(1),
+ }
+ if status:
+ summary_data.update(status)
+ log_run(
+ log_file=log_file,
+ log_data=log_data,
+ summary_file=summary_file,
+ summary_data=summary_data,
+ )
+
+if __name__ == "__main__":
+ run()
diff --git a/evaluation_harness/__init__.py b/evaluation_harness/__init__.py
new file mode 100644
index 0000000..e942c10
--- /dev/null
+++ b/evaluation_harness/__init__.py
@@ -0,0 +1,6 @@
+from .evaluators import *
+from .helper_functions import (
+ shopping_get_latest_order_url,
+ shopping_get_sku_latest_review_author,
+ shopping_get_sku_latest_review_rating,
+)
diff --git a/evaluation_harness/evaluators.py b/evaluation_harness/evaluators.py
new file mode 100644
index 0000000..527c9f9
--- /dev/null
+++ b/evaluation_harness/evaluators.py
@@ -0,0 +1,419 @@
+"""base class for evaluation"""
+# answer string match
+import collections
+import html
+import importlib
+import json
+import time
+import urllib
+from pathlib import Path
+from typing import Any, Tuple, Union, Optional
+
+from beartype import beartype
+import nltk
+nltk.download('punkt')
+from nltk.tokenize import word_tokenize # type: ignore
+
+from playwright.sync_api import CDPSession, Page
+
+from browser_env.actions import Action
+from browser_env.utils import StateInfo
+from evaluation_harness.helper_functions import (
+ PseudoPage,
+ gitlab_get_project_memeber_role,
+ llm_fuzzy_match,
+ llm_ua_match,
+ reddit_get_post_url,
+ shopping_get_latest_order_url,
+ shopping_get_sku_latest_review_author,
+ shopping_get_sku_latest_review_rating,
+)
+
+Trajectory = list[Union[Action, StateInfo]]
+
+
+class Evaluator(object):
+ def __init__(self, eval_tag: str = "") -> None:
+ self.eval_tag = eval_tag
+
+ @beartype
+ def __call__(
+ self,
+ trajectory: Trajectory,
+ config_file: Path | str,
+ page: Page | PseudoPage,
+ client: CDPSession,
+ ) -> float:
+ raise NotImplementedError
+
+ @staticmethod
+ def get_last_action(trajectory: Trajectory) -> Action:
+ try:
+ # is_bearable(trajectory[-1], Action)
+ last_action = trajectory[-1]
+ except Exception:
+ raise ValueError(
+ "The last element of trajectory should be an action, add a fake stop action if needed"
+ )
+
+ return last_action # type: ignore[return-value]
+
+ @staticmethod
+ def get_last_state(trajectory: Trajectory) -> StateInfo:
+ try:
+ # is_bearable(trajectory[-2], StateInfo)
+ last_state = trajectory[-2]
+ except Exception:
+ raise ValueError(
+ "The second last element of trajectory should be a state, add a fake stop action if needed"
+ )
+
+ return last_state # type: ignore[return-value]
+
+
+class StringEvaluator(Evaluator):
+ """Check whether the answer is correct with:
+ exact match: the answer is exactly the same as the reference answer
+ must include: each phrase in the reference answer must be included in the answer
+ fuzzy match: the answer is similar to the reference answer, using LLM judge
+ """
+
+ @staticmethod
+ @beartype
+ def clean_answer(answer: str) -> str:
+ answer = answer.strip()
+ if answer.startswith("'") and answer.endswith("'"):
+ answer = answer[1:-1]
+ elif answer.startswith('"') and answer.endswith('"'):
+ answer = answer[1:-1]
+ return answer.lower()
+
+ @staticmethod
+ @beartype
+ def exact_match(ref: str, pred: str) -> float:
+ return float(
+ StringEvaluator.clean_answer(pred)
+ == StringEvaluator.clean_answer(ref)
+ )
+
+ @staticmethod
+ @beartype
+ def must_include(ref: str, pred: str, tokenize: bool = False) -> float:
+ clean_ref = StringEvaluator.clean_answer(ref)
+ clean_pred = StringEvaluator.clean_answer(pred)
+ # tokenize the answer if the ref is a single word
+ # prevent false positive (e.g, 0)
+ if " |or| " in clean_ref or " |OR| " in clean_ref:
+ refs = clean_ref.split(" |or| ") if " |or| " in clean_ref else clean_ref.split(" |OR| ")
+ refs = [r.strip() for r in refs]
+ for r in refs:
+ if (
+ tokenize
+ and len(r) == 1
+ and len(word_tokenize(r)) == 1
+ ):
+ tok_pred = word_tokenize(r)
+ if r in tok_pred:
+ return float(r in tok_pred)
+ else:
+ if r in clean_pred:
+ return float(r in clean_pred)
+ return 0.0
+ if (
+ tokenize
+ and len(clean_ref) == 1
+ and len(word_tokenize(clean_ref)) == 1
+ ):
+ tok_pred = word_tokenize(clean_pred)
+ return float(clean_ref in tok_pred)
+ else:
+ return float(clean_ref in clean_pred)
+
+ @staticmethod
+ @beartype
+ def fuzzy_match(ref: str, pred: str, intent: str) -> float:
+ return llm_fuzzy_match(pred, ref, intent)
+
+ @staticmethod
+ @beartype
+ def ua_match(ref: str, pred: str, intent: str) -> float:
+ return llm_ua_match(pred, ref, intent)
+
+ def __call__(
+ self,
+ trajectory: Trajectory,
+ config_file: Path | str,
+ page: Page | PseudoPage | None = None,
+ client: CDPSession | None = None,
+ ) -> float:
+ with open(config_file, "r") as f:
+ configs = json.load(f)
+
+ last_action = self.get_last_action(trajectory)
+ pred = self.clean_answer(last_action["answer"])
+
+ score = 1.0
+ for approach, value in configs["eval"]["reference_answers"].items():
+ match approach:
+ case "exact_match":
+ score *= self.exact_match(ref=value, pred=pred)
+
+ case "must_include":
+ assert isinstance(value, list)
+ must_include_score = 0.
+ for must_value in value:
+ must_include_score += self.must_include(
+ ref=must_value,
+ pred=pred,
+ tokenize=(len(value) == 1),
+ )
+ must_include_score /= len(value)
+ score *= must_include_score
+ case "fuzzy_match":
+ intent = configs["intent"]
+ if value == "N/A":
+ # if the instruction only asks the model to generate N/A when encountering an unachievable task
+ # without more concrete reasons
+ score *= self.exact_match(ref=value, pred=pred)
+ # if the instruction also asks the model to generate the reason why the task is unachievable
+ # this should be the default as it will prevent false positive N/A`
+ if score != 1:
+ score = 1.0 * self.ua_match(
+ intent=configs["intent"],
+ ref=configs["eval"]["string_note"],
+ pred=pred,
+ )
+ else:
+ if isinstance(value, list):
+ fuzzy_match_value = "; ".join(value)
+ else:
+ fuzzy_match_value = value
+ fuzzy_match_score = self.fuzzy_match(
+ ref=fuzzy_match_value, pred=pred, intent=intent
+ )
+ score *= fuzzy_match_score
+ return score
+
+
+class URLEvaluator(Evaluator):
+ """Check URL matching"""
+
+ @beartype
+ def __call__(
+ self,
+ trajectory: Trajectory,
+ config_file: Path | str,
+ page: Page | PseudoPage,
+ client: CDPSession | None = None,
+ ) -> float:
+ with open(config_file, "r") as f:
+ configs = json.load(f)
+
+ def clean_url(url: str) -> str:
+ url = str(url)
+ url = url.rstrip("/")
+ url = url.lower()
+ return url
+
+ def parse_url(url: str) -> tuple[str, dict[str, list[str]]]:
+ """Parse a URL into its base, path, and query components."""
+ parsed_url = urllib.parse.urlparse(url)
+ base_path = parsed_url.netloc + parsed_url.path
+ query = urllib.parse.parse_qs(parsed_url.query)
+ return base_path, query
+
+ def parse_urls(
+ urls: list[str],
+ ) -> tuple[list[str], dict[str, set[str]]]:
+ """Parse a list of URLs."""
+ base_paths = []
+ queries = collections.defaultdict(set)
+ for url in urls:
+ base_path, query = parse_url(url)
+ base_paths.append(base_path)
+ for k, v in query.items():
+ queries[k].update(v)
+ return base_paths, queries
+
+ pred = clean_url(page.url)
+ matching_rule = configs["eval"].get("url_note", "GOLD in PRED")
+ if matching_rule == "GOLD in PRED":
+ if "or" in configs["eval"].keys():
+ or_ref_urls_list = [configs["eval"]["reference_url"]] + [item["reference_url"] for item in configs["eval"]["or"]]
+ else:
+ or_ref_urls_list = [configs["eval"]["reference_url"]]
+ or_score_list = []
+ for or_ref_urls in or_ref_urls_list:
+ ref_urls = or_ref_urls.split(" |OR| ")
+ ref_urls = [clean_url(url) for url in ref_urls]
+ ref_base_paths, ref_queries = parse_urls(ref_urls)
+ pred_base_paths, pred_query = parse_url(pred)
+
+ base_score = float(
+ any(
+ [
+ ref_base_path in pred_base_paths
+ for ref_base_path in ref_base_paths
+ ]
+ )
+ )
+ query_score = 1.0
+ for k, possible_values in ref_queries.items():
+ query_score *= float(
+ any(
+ possible_ref_value in pred_query.get(k, [])
+ for possible_ref_value in possible_values
+ )
+ )
+ or_score_list.append(base_score * query_score)
+ score = max(or_score_list)
+
+ else:
+ raise ValueError(f"Unknown matching rule: {matching_rule}")
+
+ return score
+
+
+class HTMLContentEvaluator(Evaluator):
+ """Check whether the contents appear in the page"""
+
+ @beartype
+ def __call__(
+ self,
+ trajectory: Trajectory,
+ config_file: Path | str,
+ page: Page | PseudoPage,
+ client: CDPSession | None = None,
+ ) -> float:
+ with open(config_file, "r") as f:
+ configs = json.load(f)
+
+ targets = configs["eval"]["program_html"]
+
+ score = 1.0
+ for target in targets:
+ if "or" in target.keys():
+ or_target_list = [target] + [t for t in target["or"]]
+ else:
+ or_target_list = [target]
+ or_score_list = []
+ for or_target in or_target_list:
+ target_url: str = or_target["url"] # which url to check
+ if target_url.startswith("func"):
+ func = target_url.split("func:")[1]
+ func = func.replace("__last_url__", page.url)
+ target_url = eval(func)
+
+ locator: str = or_target["locator"] # js element locator
+
+ # navigate to that url
+ if target_url != "last":
+ page.goto(target_url)
+ time.sleep(3) # TODO [shuyanzh]: fix this hard-coded sleep
+
+ # empty, use the full page
+ if not locator.strip():
+ selected_element = page.content()
+ # use JS to select the element
+ elif locator.startswith("document.") or locator.startswith(
+ "[...document."
+ ):
+ if "prep_actions" in or_target:
+ try:
+ for prep_action in or_target["prep_actions"]:
+ page.evaluate(f"() => {prep_action}")
+ except Exception:
+ pass
+ try:
+ selected_element = str(page.evaluate(f"() => {locator}"))
+ if not selected_element:
+ selected_element = ""
+ except Exception:
+ # the page is wrong, return empty
+ selected_element = ""
+ # run program to call API
+ elif locator.startswith("func:"): # a helper function
+ func = locator.split("func:")[1]
+ func = func.replace("__page__", "page")
+ selected_element = eval(func)
+ else:
+ raise ValueError(f"Unknown locator: {locator}")
+
+ selected_element = html.unescape(selected_element)
+
+ if "exact_match" in or_target["required_contents"]:
+ required_contents = or_target["required_contents"]["exact_match"]
+ cur_score = StringEvaluator.exact_match(
+ ref=required_contents, pred=selected_element
+ )
+ or_score_list.append(cur_score)
+ print(f"[exact match] {cur_score}, selected element: {selected_element}, required contents: {required_contents}")
+ elif "must_include" in or_target["required_contents"]:
+ required_contents = or_target["required_contents"]["must_include"]
+ assert isinstance(required_contents, list)
+ content_score_list = []
+ for content in required_contents:
+ content_or = content.split(" |OR| ")
+ cur_score = any(
+ [
+ StringEvaluator.must_include(
+ ref=content,
+ pred=selected_element,
+ tokenize=False,
+ )
+ for content in content_or
+ ]
+ )
+ content_score_list.append(cur_score)
+ # score *= float(cur_score)
+ print(f"[must include] {cur_score}, selected element: {selected_element}, required contents: {content_or}")
+ or_score_list.append(sum(content_score_list)/len(content_score_list))
+ else:
+ raise ValueError(
+ f"Unknown required_contents: {or_target['required_contents'].keys()}"
+ )
+ or_score = max(or_score_list)
+ score *= or_score
+
+ return score
+
+
+class EvaluatorComb:
+ def __init__(self, evaluators: list[Evaluator]) -> None:
+ self.evaluators = evaluators
+
+ @beartype
+ def __call__(
+ self,
+ trajectory: Trajectory,
+ config_file: Path | str,
+ page: Optional[Page | PseudoPage] = None,
+ client: Optional[CDPSession] = None,
+ ) -> float:
+ score = 1.0
+ for evaluator in self.evaluators:
+ cur_score = evaluator(trajectory, config_file, page, client)
+ score *= cur_score
+ return score
+
+
+@beartype
+def evaluator_router(config_file: Path | str) -> EvaluatorComb:
+ """Router to get the evaluator class"""
+ with open(config_file, "r") as f:
+ configs = json.load(f)
+
+ eval_types = configs["eval"]["eval_types"]
+ evaluators: list[Evaluator] = []
+ for eval_type in eval_types:
+ match eval_type:
+ case "string_match":
+ evaluators.append(StringEvaluator())
+ case "url_match":
+ evaluators.append(URLEvaluator())
+ case "program_html":
+ evaluators.append(HTMLContentEvaluator())
+ case _:
+ raise ValueError(f"eval_type {eval_type} is not supported")
+
+ return EvaluatorComb(evaluators)
diff --git a/evaluation_harness/helper_functions.py b/evaluation_harness/helper_functions.py
new file mode 100644
index 0000000..d1a55cf
--- /dev/null
+++ b/evaluation_harness/helper_functions.py
@@ -0,0 +1,220 @@
+"""Implements helper functions to assist evaluation cases where other evaluators are not suitable."""
+import json
+from typing import Any
+from urllib.parse import urlparse
+
+import requests
+from playwright.sync_api import CDPSession, Page
+
+from browser_env.env_config import (
+ ACCOUNTS,
+ GITLAB,
+ MAP,
+ REDDIT,
+ SHOPPING,
+ SHOPPING_ADMIN,
+ WIKIPEDIA,
+)
+from llms.providers.openai_utils import (
+ generate_from_openai_chat_completion,
+)
+
+
+def shopping_get_auth_token() -> str:
+ response = requests.post(
+ url=f"{SHOPPING}/rest/default/V1/integration/admin/token",
+ headers={"content-type": "application/json"},
+ data=json.dumps(
+ {
+ "username": ACCOUNTS["shopping_site_admin"]["username"],
+ "password": ACCOUNTS["shopping_site_admin"]["password"],
+ }
+ ),
+ )
+ token: str = response.json()
+ return token
+
+
+def shopping_get_latest_order_url() -> str:
+ """Get the latest order url from the shopping website."""
+
+ header = {
+ "Authorization": f"Bearer {shopping_get_auth_token()}",
+ "Content-Type": "application/json",
+ }
+
+ params = {
+ "searchCriteria[sortOrders][0][field]": "created_at",
+ "searchCriteria[sortOrders][0][direction]": "DESC",
+ "searchCriteria[pageSize]": "1",
+ }
+
+ response = requests.get(
+ f"{SHOPPING}/rest/V1/orders", params=params, headers=header
+ )
+ assert response.status_code == 200
+ response_obj = response.json()["items"][0]
+ order_id = int(response_obj["increment_id"])
+ order_url = f"{SHOPPING}/sales/order/view/order_id/{order_id}/"
+ return order_url
+
+
+def shopping_get_sku_latest_review_author(sku: str) -> str:
+ """Get the latest review for shopping admin."""
+ header = {
+ "Authorization": f"Bearer {shopping_get_auth_token()}",
+ "Content-Type": "application/json",
+ }
+ response = requests.get(
+ f"{SHOPPING}/rest/V1/products/{sku}/reviews", headers=header
+ )
+ assert response.status_code == 200
+ response_obj = response.json()
+ if len(response_obj) == 0:
+ return ""
+ author: str = response_obj[-1]["nickname"]
+ return author
+
+
+def shopping_get_sku_latest_review_rating(sku: str) -> str:
+ """Get the latest review for shopping admin."""
+ header = {
+ "Authorization": f"Bearer {shopping_get_auth_token()}",
+ "Content-Type": "application/json",
+ }
+ response = requests.get(
+ f"{SHOPPING}/rest/V1/products/{sku}/reviews", headers=header
+ )
+ assert response.status_code == 200
+ response_obj = response.json()
+ if len(response_obj) == 0:
+ return ""
+ assert response_obj[0]["ratings"][0]["rating_name"] == "Rating"
+ rating: str = str(response_obj[-1]["ratings"][0]["percent"])
+ return rating
+
+
+def reddit_get_post_url(url: str) -> str:
+ """Get the post url"""
+ # Url is http://domain/f/subreddit/post_id/...
+ # get domain, subreddit, post_id
+ domain = urlparse(url).netloc
+ tok_url = urlparse(url).path.split("/")
+ # not a valid post/comment url, return the url as is
+ if len(tok_url) < 4:
+ return url
+ if tok_url[1] != "f":
+ return url
+ subreddit = urlparse(url).path.split("/")[2]
+ post_id = urlparse(url).path.split("/")[3]
+ scheme = urlparse(url).scheme
+ post_url = f"{scheme}://{domain}/f/{subreddit}/{post_id}/"
+ return post_url
+
+
+def gitlab_get_project_memeber_role(page: Page, account_name: str) -> str:
+ # get the account index
+ try:
+ account_idx = page.evaluate(
+ f"""(() => {{
+ const elements = document.querySelectorAll("td[data-label='Account'] span.gl-avatar-labeled-sublabel");
+ let index = -1; // Default value if not found
+
+ for(let i = 0; i < elements.length; i++) {{
+ if(elements[i].outerText === '@{account_name}') {{
+ index = i;
+ break;
+ }}
+ }}
+
+ return index;
+ }})()"""
+ )
+
+ # get the role
+ role: str = page.evaluate(
+ f"""(() => {{
+ return document.querySelectorAll("td.col-max-role span")[{account_idx}].outerText;
+ }})()"""
+ )
+ except Exception:
+ role = ""
+
+ return role
+
+
+def llm_fuzzy_match(pred: str, reference: str, question: str) -> float:
+ """Check whether the prediction matches the reference with GPT4-turbo"""
+ messages: list[dict[str, Any]] = []
+ # construct the question to ask
+ message = "Help a teacher to grade the answer of a student given a question. Keep in mind that the student has performed the action to get the answer. They are allowed to use different phrasing or wording to answer the question. The goal is to evaluate whether the key points in the reference answer are included in the student's answer. We allow answers with additional information that doesn't contradict the reference answer and review them as fully (not partially) correct.\n"
+ message += f"question: {question}\n"
+ message += f"reference answer: {reference}\n"
+ message += "all the string 'N/A' that you see is a special sequence that means 'not achievable'\n"
+ message += f"student answer: {pred}\n"
+ message += "Conclude the judgement by correct/incorrect/partially correct and explain why."
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant"},
+ {"role": "user", "content": message},
+ ]
+ response = generate_from_openai_chat_completion(
+ model="gpt-4-1106-preview",
+ messages=messages,
+ temperature=0,
+ max_tokens=768,
+ top_p=1.0,
+ context_length=0,
+ ).lower()
+ print(response)
+ if "partially correct" in response or "incorrect" in response:
+ return 0.0
+ else:
+ assert "correct" in response
+ return 1.0
+
+def llm_ua_match(pred: str, reference: str, question: str) -> float:
+ """Check whether the prediction matches the reference with GPT-turbo"""
+ messages: list[dict[str, Any]] = []
+ # construct the question to ask
+ message = ""
+ message += f"task: {question}\n"
+ message += f"actual unachievable reason: {reference}\n"
+ message += f"reported unachievable reason: {pred}\n"
+ message += (
+ "The task described above is inherently unachievable due to the reason specified under 'actual unachievable reason'. "
+ "An individual previously attempted this task and was unable to complete it. They provided a reason for their failure, "
+ "which is listed under 'reported unachievable reason'. Your role is to review both the actual and reported reasons. "
+ "Determine if the reported reason aligns with the actual reason, even if implicitly. "
+ "If the stated reason is in line with the actual reason, respond with 'same'. Otherwise, respond with 'different'."
+ )
+ messages = [
+ {"role": "system", "content": "You are a helpful assistant"},
+ {"role": "user", "content": message},
+ ]
+
+ response = generate_from_openai_chat_completion(
+ model="gpt-4-1106-preview",
+ messages=messages,
+ temperature=0,
+ max_tokens=768,
+ top_p=1.0,
+ context_length=0,
+ ).lower()
+ if "different" in response:
+ return 0.0
+ else:
+ assert "same" in response
+ return 1.0
+
+
+class PseudoPage:
+ def __init__(self, original_page: Page, url: str):
+ self.url = url
+ self.original_page = original_page
+
+ def __getattr__(self, attr: str) -> Any:
+ # Delegate attribute access to the original page object
+ if attr not in ["url"]:
+ return getattr(self.original_page, attr)
+ else:
+ return getattr(self, attr)
diff --git a/files/overview.png b/files/overview.png
new file mode 100644
index 0000000..15f09da
Binary files /dev/null and b/files/overview.png differ
diff --git a/llms/__init__.py b/llms/__init__.py
new file mode 100644
index 0000000..aac6b24
--- /dev/null
+++ b/llms/__init__.py
@@ -0,0 +1,16 @@
+"""This module is adapt from https://github.com/zeno-ml/zeno-build"""
+from .providers.hf_utils import generate_from_huggingface_completion
+from .providers.openai_utils import (
+ generate_from_openai_chat_completion,
+ generate_from_openai_completion,
+)
+from .providers.anthropic_utils import generate_from_anthropic_completion
+from .utils import call_llm
+
+__all__ = [
+ "generate_from_openai_completion",
+ "generate_from_openai_chat_completion",
+ "generate_from_huggingface_completion",
+ "call_llm",
+ "generate_from_anthropic_completion"
+]
diff --git a/llms/lm_config.py b/llms/lm_config.py
new file mode 100644
index 0000000..13ec79f
--- /dev/null
+++ b/llms/lm_config.py
@@ -0,0 +1,61 @@
+"""Config for language models."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class LMConfig:
+ """A config for a language model.
+
+ Attributes:
+ provider: The name of the API provider.
+ model: The name of the model.
+ model_cls: The Python class corresponding to the model, mostly for
+ Hugging Face transformers.
+ tokenizer_cls: The Python class corresponding to the tokenizer, mostly
+ for Hugging Face transformers.
+ mode: The mode of the API calls, e.g., "chat" or "generation".
+ """
+
+ provider: str
+ model: str
+ model_cls: type | None = None
+ tokenizer_cls: type | None = None
+ mode: str | None = None
+ gen_config: dict[str, Any] = dataclasses.field(default_factory=dict)
+ cuda: str = '0'
+
+
+def construct_llm_config(args: argparse.Namespace) -> LMConfig:
+ llm_config = LMConfig(
+ provider=args.provider, model=args.model, mode=args.mode, cuda=args.cuda
+ )
+ if args.provider == "openai":
+ llm_config.gen_config["temperature"] = args.temperature
+ llm_config.gen_config["top_p"] = args.top_p
+ llm_config.gen_config["context_length"] = args.context_length
+ llm_config.gen_config["max_tokens"] = args.max_tokens
+ llm_config.gen_config["stop_token"] = args.stop_token
+ llm_config.gen_config["max_obs_length"] = args.max_obs_length
+ llm_config.gen_config["max_retry"] = args.max_retry
+ elif args.provider == "huggingface":
+ llm_config.gen_config["temperature"] = args.temperature
+ llm_config.gen_config["top_p"] = args.top_p
+ llm_config.gen_config["max_new_tokens"] = args.max_tokens
+ llm_config.gen_config["stop_sequences"] = (
+ [args.stop_token] if args.stop_token else None
+ )
+ llm_config.gen_config["max_obs_length"] = args.max_obs_length
+ llm_config.gen_config["model_endpoint"] = args.model_endpoint
+ llm_config.gen_config["max_retry"] = args.max_retry
+ elif args.provider == "anthropic":
+ llm_config.gen_config["max_obs_length"] = None
+ llm_config.gen_config["max_retry"] = args.max_retry
+ else:
+ raise NotImplementedError(f"provider {args.provider} not implemented")
+ return llm_config
diff --git a/llms/providers/anthropic_utils.py b/llms/providers/anthropic_utils.py
new file mode 100644
index 0000000..754c77f
--- /dev/null
+++ b/llms/providers/anthropic_utils.py
@@ -0,0 +1,45 @@
+# Use the native inference API to send a text message to Anthropic Claude.
+
+import boto3
+import json
+
+def generate_from_anthropic_completion(
+ prompt: str,
+ engine: str,
+) -> str:
+ # Create a Bedrock Runtime client in the AWS Region of your choice.
+ client = boto3.client("bedrock-runtime", region_name="us-east-1")
+
+ # Set the model ID, e.g., Claude 3 Haiku.
+ model_id = "anthropic.claude-3-sonnet-20240229-v1:0"
+
+ # Format the request payload using the model's native structure.
+ native_request = {
+ "anthropic_version": "bedrock-2023-05-31",
+ "max_tokens": 1024,
+ "temperature": 0.5,
+ "messages": [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": prompt}],
+ }
+ ],
+ }
+
+ # Convert the native request to JSON.
+ request = json.dumps(native_request)
+
+ try:
+ # Invoke the model with the request.
+ response = client.invoke_model(modelId=engine, body=request)
+
+ except Exception as e: # (ClientError, Exception) as e:
+ raise KeyError(f"ERROR: Can't invoke '{engine}'. Reason: {e}")
+
+ # Decode the response body.
+ model_response = json.loads(response["body"].read())
+
+ # Extract and print the response text.
+ response_text = model_response["content"][0]["text"]
+
+ return response_text
\ No newline at end of file
diff --git a/llms/providers/hf_utils.py b/llms/providers/hf_utils.py
new file mode 100644
index 0000000..b5e8987
--- /dev/null
+++ b/llms/providers/hf_utils.py
@@ -0,0 +1,21 @@
+from text_generation import Client # type: ignore
+
+
+def generate_from_huggingface_completion(
+ prompt: str,
+ model_endpoint: str,
+ temperature: float,
+ top_p: float,
+ max_new_tokens: int,
+ stop_sequences: list[str] | None = None,
+) -> str:
+ client = Client(model_endpoint, timeout=60)
+ generation: str = client.generate(
+ prompt=prompt,
+ temperature=temperature,
+ top_p=top_p,
+ max_new_tokens=max_new_tokens,
+ stop_sequences=stop_sequences,
+ ).generated_text
+
+ return generation
diff --git a/llms/providers/openai_utils.py b/llms/providers/openai_utils.py
new file mode 100644
index 0000000..fb86d52
--- /dev/null
+++ b/llms/providers/openai_utils.py
@@ -0,0 +1,283 @@
+"""Tools to generate from OpenAI prompts.
+Adopted from https://github.com/zeno-ml/zeno-build/"""
+
+import asyncio
+import logging
+import os
+import random
+import time
+from typing import Any
+
+import aiolimiter
+import openai
+from openai import OpenAI
+from tqdm.asyncio import tqdm_asyncio
+
+
+def retry_with_exponential_backoff( # type: ignore
+ func,
+ initial_delay: float = 1,
+ exponential_base: float = 2,
+ jitter: bool = True,
+ max_retries: int = 3,
+ errors: tuple[Any] = (openai.RateLimitError,),
+):
+ """Retry a function with exponential backoff."""
+
+ def wrapper(*args, **kwargs): # type: ignore
+ # Initialize variables
+ num_retries = 0
+ delay = initial_delay
+
+ # Loop until a successful response or max_retries is hit or an exception is raised
+ while True:
+ try:
+ return func(*args, **kwargs)
+ # Retry on specified errors
+ except errors as e:
+ # Increment retries
+ num_retries += 1
+
+ # Check if max retries has been reached
+ if num_retries > max_retries:
+ raise Exception(
+ f"Maximum number of retries ({max_retries}) exceeded."
+ )
+
+ # Increment the delay
+ delay *= exponential_base * (1 + jitter * random.random())
+ print(f"Retrying in {delay} seconds.")
+ # Sleep for the delay
+ time.sleep(delay)
+
+ # Raise exceptions for any errors not specified
+ except Exception as e:
+ raise e
+
+ return wrapper
+
+
+async def _throttled_openai_completion_acreate(
+ engine: str,
+ prompt: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ limiter: aiolimiter.AsyncLimiter,
+) -> dict[str, Any]:
+ async with limiter:
+ for _ in range(3):
+ try:
+ return await openai.Completion.acreate( # type: ignore
+ engine=engine,
+ prompt=prompt,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ )
+ except openai.RateLimitError:
+ logging.warning(
+ "OpenAI API rate limit exceeded. Sleeping for 10 seconds."
+ )
+ await asyncio.sleep(10)
+ except openai.APIError as e:
+ logging.warning(f"OpenAI API error: {e}")
+ break
+ return {"choices": [{"message": {"content": ""}}]}
+
+
+async def agenerate_from_openai_completion(
+ prompts: list[str],
+ engine: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ context_length: int,
+ requests_per_minute: int = 300,
+) -> list[str]:
+ """Generate from OpenAI Completion API.
+
+ Args:
+ prompts: list of prompts
+ temperature: Temperature to use.
+ max_tokens: Maximum number of tokens to generate.
+ top_p: Top p to use.
+ context_length: Length of context to use.
+ requests_per_minute: Number of requests per minute to allow.
+
+ Returns:
+ List of generated responses.
+ """
+ if "OPENAI_API_KEY" not in os.environ:
+ raise ValueError(
+ "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+ )
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+ openai.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+ limiter = aiolimiter.AsyncLimiter(requests_per_minute)
+ async_responses = [
+ _throttled_openai_completion_acreate(
+ engine=engine,
+ prompt=prompt,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ limiter=limiter,
+ )
+ for prompt in prompts
+ ]
+ responses = await tqdm_asyncio.gather(*async_responses)
+ return [x["choices"][0]["text"] for x in responses]
+
+
+@retry_with_exponential_backoff
+def generate_from_openai_completion(
+ prompt: str,
+ engine: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ stop_token: str | None = None,
+) -> str:
+ if "OPENAI_API_KEY" not in os.environ:
+ raise ValueError(
+ "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+ )
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+ openai.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+ response = OpenAI().completions.create( # type: ignore
+ prompt=prompt,
+ engine=engine,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ stop=[stop_token],
+ )
+ answer: str = response["choices"][0]["text"]
+ return answer
+
+
+async def _throttled_openai_chat_completion_acreate(
+ model: str,
+ messages: list[dict[str, str]],
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ limiter: aiolimiter.AsyncLimiter,
+) -> dict[str, Any]:
+ async with limiter:
+ for _ in range(3):
+ try:
+ return await openai.ChatCompletion.acreate( # type: ignore
+ model=model,
+ messages=messages,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ )
+ except openai.RateLimitError:
+ logging.warning(
+ "OpenAI API rate limit exceeded. Sleeping for 10 seconds."
+ )
+ await asyncio.sleep(10)
+ except asyncio.exceptions.TimeoutError:
+ logging.warning("OpenAI API timeout. Sleeping for 10 seconds.")
+ await asyncio.sleep(10)
+ except openai.APIError as e:
+ logging.warning(f"OpenAI API error: {e}")
+ break
+ return {"choices": [{"message": {"content": ""}}]}
+
+
+async def agenerate_from_openai_chat_completion(
+ messages_list: list[list[dict[str, str]]],
+ engine: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ context_length: int,
+ requests_per_minute: int = 300,
+) -> list[str]:
+ """Generate from OpenAI Chat Completion API.
+
+ Args:
+ messages_list: list of message list
+ temperature: Temperature to use.
+ max_tokens: Maximum number of tokens to generate.
+ top_p: Top p to use.
+ context_length: Length of context to use.
+ requests_per_minute: Number of requests per minute to allow.
+
+ Returns:
+ List of generated responses.
+ """
+ if "OPENAI_API_KEY" not in os.environ:
+ raise ValueError(
+ "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+ )
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+ openai.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+ limiter = aiolimiter.AsyncLimiter(requests_per_minute)
+ async_responses = [
+ _throttled_openai_chat_completion_acreate(
+ model=engine,
+ messages=message,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ limiter=limiter,
+ )
+ for message in messages_list
+ ]
+ responses = await tqdm_asyncio.gather(*async_responses)
+ return [x["choices"][0]["message"]["content"] for x in responses]
+
+
+@retry_with_exponential_backoff
+def generate_from_openai_chat_completion(
+ messages: list[dict[str, str]],
+ model: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ context_length: int,
+ stop_token: str | None = None,
+) -> str:
+ if "OPENAI_API_KEY" not in os.environ:
+ raise ValueError(
+ "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+ )
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+ openai.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+
+ response = OpenAI().chat.completions.create( # type: ignore
+ model=model,
+ messages=messages,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ top_p=top_p,
+ stop=[stop_token] if stop_token else None,
+ )
+ answer: str = response.choices[0].message.content
+ return answer
+
+
+@retry_with_exponential_backoff
+# debug only
+def fake_generate_from_openai_chat_completion(
+ messages: list[dict[str, str]],
+ model: str,
+ temperature: float,
+ max_tokens: int,
+ top_p: float,
+ context_length: int,
+ stop_token: str | None = None,
+) -> str:
+ if "OPENAI_API_KEY" not in os.environ:
+ raise ValueError(
+ "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+ )
+ openai.api_key = os.environ["OPENAI_API_KEY"]
+ openai.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+ answer = "Let's think step-by-step. This page shows a list of links and buttons. There is a search box with the label 'Search query'. I will click on the search box to type the query. So the action I will perform is \"click [60]\"."
+ return answer
diff --git a/llms/tokenizers.py b/llms/tokenizers.py
new file mode 100644
index 0000000..9883ed1
--- /dev/null
+++ b/llms/tokenizers.py
@@ -0,0 +1,29 @@
+from typing import Any
+
+import tiktoken
+from transformers import LlamaTokenizer # type: ignore
+
+
+class Tokenizer(object):
+ def __init__(self, provider: str, model_name: str) -> None:
+ if provider == "openai":
+ self.tokenizer = tiktoken.encoding_for_model(model_name)
+ elif provider == "huggingface":
+ self.tokenizer = LlamaTokenizer.from_pretrained(model_name)
+ # turn off adding special tokens automatically
+ self.tokenizer.add_special_tokens = False # type: ignore[attr-defined]
+ self.tokenizer.add_bos_token = False # type: ignore[attr-defined]
+ self.tokenizer.add_eos_token = False # type: ignore[attr-defined]
+ elif provider == "anthropic":
+ pass
+ else:
+ raise NotImplementedError
+
+ def encode(self, text: str) -> list[int]:
+ return self.tokenizer.encode(text)
+
+ def decode(self, ids: list[int]) -> str:
+ return self.tokenizer.decode(ids)
+
+ def __call__(self, text: str) -> list[int]:
+ return self.tokenizer.encode(text)
diff --git a/llms/utils.py b/llms/utils.py
new file mode 100644
index 0000000..cf3cc52
--- /dev/null
+++ b/llms/utils.py
@@ -0,0 +1,67 @@
+import argparse
+from typing import Any
+from transformers import AutoTokenizer, AutoModel
+
+from llms import (
+ generate_from_huggingface_completion,
+ generate_from_openai_chat_completion,
+ generate_from_openai_completion,
+ lm_config,
+)
+
+APIInput = str | list[Any] | dict[str, Any]
+
+model = None
+tokenizer = None
+
+def call_llm(
+ lm_config: lm_config.LMConfig,
+ prompt: APIInput,
+) -> str:
+ global model
+ global tokenizer
+
+ response: str
+
+ if lm_config.provider == "openai":
+ if lm_config.mode == "chat":
+ assert isinstance(prompt, list)
+ response = generate_from_openai_chat_completion(
+ messages=prompt,
+ model=lm_config.model,
+ temperature=lm_config.gen_config["temperature"],
+ top_p=lm_config.gen_config["top_p"],
+ context_length=lm_config.gen_config["context_length"],
+ max_tokens=lm_config.gen_config["max_tokens"],
+ stop_token=None,
+ )
+ elif lm_config.mode == "completion":
+ assert isinstance(prompt, str)
+ response = generate_from_openai_completion(
+ prompt=prompt,
+ engine=lm_config.model,
+ temperature=lm_config.gen_config["temperature"],
+ max_tokens=lm_config.gen_config["max_tokens"],
+ top_p=lm_config.gen_config["top_p"],
+ stop_token=lm_config.gen_config["stop_token"],
+ )
+ else:
+ raise ValueError(
+ f"OpenAI models do not support mode {lm_config.mode}"
+ )
+ elif lm_config.provider == "huggingface":
+ assert isinstance(prompt, str)
+ response = generate_from_huggingface_completion(
+ prompt=prompt,
+ model_endpoint=lm_config.gen_config["model_endpoint"],
+ temperature=lm_config.gen_config["temperature"],
+ top_p=lm_config.gen_config["top_p"],
+ stop_sequences=lm_config.gen_config["stop_sequences"],
+ max_new_tokens=lm_config.gen_config["max_new_tokens"],
+ )
+ else:
+ raise NotImplementedError(
+ f"Provider {lm_config.provider} not implemented"
+ )
+
+ return response
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..f85e701
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,11 @@
+lxml
+torch
+ctranslate2
+boto3
+accelerate
+bitsandbytes
+peft
+matplotlib
+google
+google-generativeai
+google-ai-generativelanguage
\ No newline at end of file
diff --git a/scripts/run_config.sh b/scripts/run_config.sh
new file mode 100644
index 0000000..44d095e
--- /dev/null
+++ b/scripts/run_config.sh
@@ -0,0 +1,11 @@
+export SHOPPING="http://:7770"
+export SHOPPING_ADMIN="http://:7780/admin"
+export REDDIT="http://:9999"
+export GITLAB="http://:8023"
+export MAP="http://:3000"
+export WIKIPEDIA="http://:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing"
+export HOMEPAGE="http://:4399"
+export OPENAI_API_KEY=
+conda activate webarena
+python browser_env/auto_login.py
+python eval_webarena.py --config AgentOccam/configs/AgentOccam.yml
\ No newline at end of file
diff --git a/scripts/run_webarena.sh b/scripts/run_webarena.sh
new file mode 100644
index 0000000..b318292
--- /dev/null
+++ b/scripts/run_webarena.sh
@@ -0,0 +1,6 @@
+python webarena_replication.py \
+ --instruction_path webarena/agent/prompts/jsons/p_cot_id_actree_2s.json \
+ --test_start_idx 65 \
+ --test_end_idx 66 \
+ --model gpt-4-turbo \
+ --result_dir ../AgentOccam-Trajectories/WebArena-replication
\ No newline at end of file
diff --git a/webagents_step/__init__.py b/webagents_step/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webagents_step/agents/agent.py b/webagents_step/agents/agent.py
new file mode 100644
index 0000000..4a0670b
--- /dev/null
+++ b/webagents_step/agents/agent.py
@@ -0,0 +1,106 @@
+from typing import List
+
+
+class Agent:
+ def __init__(
+ self,
+ max_actions,
+ verbose=0,
+ logging=False,
+ previous_actions: List = None,
+ previous_reasons: List = None,
+ previous_responses: List = None,
+ ):
+ self.previous_actions = [] if previous_actions is None else previous_actions
+ self.previous_reasons = [] if previous_reasons is None else previous_reasons
+ self.previous_responses = [] if previous_responses is None else previous_responses
+ self.max_actions = max_actions
+ self.verbose = verbose
+ self.logging = logging
+ self.trajectory = []
+ self.data_to_log = {}
+
+ def reset(self):
+ self.previous_actions = []
+ self.previous_reasons = []
+ self.previous_responses = []
+ self.trajectory = []
+ self.data_to_log = {}
+
+ def get_trajectory(self):
+ return self.trajectory
+
+ def update_history(self, action, reason):
+ if action:
+ self.previous_actions += [action]
+ if reason:
+ self.previous_reasons += [reason]
+
+ def predict_action(self, objective, observation, url=None):
+ pass
+
+ def receive_response(self, response):
+ self.previous_responses += [response]
+
+ def act(self, objective, env):
+ while not env.done():
+ observation = env.observation()
+ action, reason = self.predict_action(
+ objective=objective, observation=observation, url=env.get_url()
+ )
+ status = env.step(action)
+
+ if self.logging:
+ self.log_step(
+ objective=objective,
+ url=env.get_url(),
+ observation=observation,
+ action=action,
+ reason=reason,
+ status=status,
+ )
+
+ if len(self.previous_actions) >= self.max_actions:
+ print(f"Agent exceeded max actions: {self.max_actions}")
+ break
+
+ return status
+
+ async def async_act(self, objective, env):
+ while not env.done():
+ observation = await env.observation()
+ action, reason = self.predict_action(
+ objective=objective, observation=observation, url=env.get_url()
+ )
+ status = await env.step(action)
+
+ if self.logging:
+ self.log_step(
+ objective=objective,
+ url=env.get_url(),
+ observation=observation,
+ action=action,
+ reason=reason,
+ status=status,
+ )
+
+ if len(self.previous_actions) >= self.max_actions:
+ print(f"Agent exceeded max actions: {self.max_actions}")
+ break
+
+ return status
+
+ def log_step(self, objective, url, observation, action, reason, status):
+ self.data_to_log['objective'] = objective
+ self.data_to_log['url'] = url
+ self.data_to_log['observation'] = observation if isinstance(observation, str) else observation["text"]
+ self.data_to_log['previous_actions'] = self.previous_actions[:-1]
+ self.data_to_log['previous_responses'] = self.previous_responses[:-1]
+ self.data_to_log['previous_reasons'] = self.previous_reasons[:-1]
+ self.data_to_log['action'] = action
+ self.data_to_log['reason'] = reason
+ if status:
+ for (k, v) in status.items():
+ self.data_to_log[k] = v
+ self.trajectory.append(self.data_to_log)
+ self.data_to_log = {}
diff --git a/webagents_step/agents/prompt_agent.py b/webagents_step/agents/prompt_agent.py
new file mode 100644
index 0000000..ced63b3
--- /dev/null
+++ b/webagents_step/agents/prompt_agent.py
@@ -0,0 +1,59 @@
+from webagents_step.agents.agent import Agent
+from typing import List
+from webagents_step.utils.llm import fill_prompt_template, construct_llm_message_openai, call_openai_llm, parse_action_reason, calculate_cost_openai, call_anthropic_llm
+
+class PromptAgent(Agent):
+ def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False,
+ debug: bool = False, prompt_template: str = None, model: str = "gpt-3.5-turbo",
+ prompt_mode: str = "chat", previous_actions: List = None, previous_reasons: List = None, previous_responses: List = None):
+ super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions, previous_reasons=previous_reasons, previous_responses=previous_responses)
+ self.debug = debug
+ self.prompt_template = prompt_template
+ self.model = model
+ self.prompt_mode = prompt_mode
+
+ def previous_history(self):
+ previous_history = []
+
+ if len(self.previous_actions) == len(self.previous_responses):
+ for action, response in zip(self.previous_actions, self.previous_responses):
+ if response:
+ previous_history.append(f"{response} = {action}")
+ else:
+ previous_history.append(action)
+ previous_history="\n".join(previous_history)
+ else:
+ previous_history = "\n".join(action for action in self.previous_actions if action is not None) if self.previous_actions is not None else ""
+
+
+ return previous_history
+
+ def predict_action(self, objective, observation, url=None):
+ prompt = fill_prompt_template(prompt_template=self.prompt_template, objective=objective,
+ observation=observation, url=url,
+ previous_history=self.previous_history())
+ messages = construct_llm_message_openai(prompt=prompt, prompt_mode=self.prompt_mode)
+ model_response = call_openai_llm(messages=messages, model=self.model)
+ action, reason = parse_action_reason(model_response)
+
+ if self.logging:
+ self.data_to_log['prompt'] = messages
+
+ if self.verbose > 0:
+ print(f"\n OBJECTIVE: {objective}")
+ print(f"\n URL: {url}")
+ print(f"\n PREVIOUS HISTORY: {self.previous_history()}")
+ print(f"\n REASON: {reason}")
+ print(f"\n ACTION: {action}")
+ if self.verbose > 1:
+ print(f"\n OBSERVATION: {observation}")
+ print(f"\n RESPONSE: {model_response}")
+
+ if self.debug:
+ human_input = input()
+ if human_input != "c":
+ action = human_input
+ reason = "None"
+
+ self.update_history(action=action, reason=reason)
+ return action, reason
\ No newline at end of file
diff --git a/webagents_step/agents/step_agent.py b/webagents_step/agents/step_agent.py
new file mode 100644
index 0000000..bd3482c
--- /dev/null
+++ b/webagents_step/agents/step_agent.py
@@ -0,0 +1,105 @@
+from webagents_step.agents.agent import Agent
+from webagents_step.utils.stack import Stack
+from webagents_step.agents.prompt_agent import PromptAgent
+
+from typing import List, Dict
+import re
+import time
+
+class StepAgent(Agent):
+ def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False,
+ debug: bool = False,
+ root_action: str = None,
+ action_to_prompt_dict: Dict = None,
+ low_level_action_list: List = None,
+ model: str = "gpt-3.5-turbo",
+ prompt_mode: str = "chat", previous_actions: List = None):
+ super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions)
+ self.debug = debug
+ self.root_action = root_action
+ self.action_to_prompt_dict = {} if action_to_prompt_dict is None else action_to_prompt_dict
+ self.low_level_action_list = [] if low_level_action_list is None else low_level_action_list
+ self.model = model
+ self.prompt_mode = prompt_mode
+ self.stack = Stack()
+
+ def is_done(self, action):
+ if action and "stop" in action:
+ return True
+ return False
+
+ def is_low_level_action(self, action):
+ if not action:
+ return False
+ action_type = action.split()[0]
+ return (action_type in self.low_level_action_list)
+
+ def is_high_level_action(self, action):
+ if not action:
+ return False
+ action_type = action.split()[0]
+ return (action_type in self.action_to_prompt_dict)
+
+ def init_root_agent(self, objective):
+ root_prompt_template = self.action_to_prompt_dict[self.root_action]
+ agent = PromptAgent(
+ prompt_template=root_prompt_template,
+ model=self.model,
+ prompt_mode=self.prompt_mode,
+ max_actions=self.max_actions,
+ verbose=self.verbose,
+ logging=self.logging,
+ debug=self.debug,
+ previous_actions=[],
+ previous_reasons=[],
+ previous_responses=[]
+ )
+ return {'agent': agent, 'objective': objective}
+
+ def init_agent(self, action):
+ pattern = r'(\w+)\s+\[(.*?)\]'
+ matches = re.findall(pattern, action)
+ action_type, _ = matches[0]
+ objective = action
+ prompt_template = self.action_to_prompt_dict[action_type]
+ agent = PromptAgent(
+ prompt_template=prompt_template,
+ model=self.model,
+ prompt_mode=self.prompt_mode,
+ max_actions=self.max_actions,
+ verbose=self.verbose,
+ logging=self.logging,
+ debug=self.debug,
+ previous_actions=[],
+ previous_reasons=[],
+ previous_responses=[]
+ )
+ return {'agent': agent, 'objective': objective}
+
+ def predict_action(self, objective, observation, url=None):
+ observation = observation["text"] if isinstance(observation, dict) else observation
+ if self.stack.is_empty():
+ new_element = self.init_root_agent(objective=objective)
+ self.stack.push(new_element)
+
+ action, reason = None, None
+ while not self.stack.is_empty():
+ element = self.stack.peek()
+ action, reason = element['agent'].predict_action(objective=element['objective'], observation=observation, url=url)
+ if (not self.is_done(action)) and self.is_low_level_action(action):
+ element['agent'].receive_response("")
+ return action, reason
+ if (not self.is_done(action)) and self.is_high_level_action(action):
+ new_element = self.init_agent(action)
+ self.stack.push(new_element)
+ if self.logging:
+ self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={})
+ continue
+ if self.is_done(action):
+ self.stack.pop()
+ if not self.stack.is_empty():
+ self.stack.peek()['agent'].receive_response(re.search(r"\[(.*?)\]", action).group(1))
+ if self.logging:
+ self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={})
+ continue
+ return action, reason
\ No newline at end of file
diff --git a/webagents_step/environment/env.py b/webagents_step/environment/env.py
new file mode 100644
index 0000000..949ef07
--- /dev/null
+++ b/webagents_step/environment/env.py
@@ -0,0 +1,18 @@
+class WebEnvironment():
+ def __init__(self):
+ pass
+
+ def reset(self):
+ pass
+
+ def observation(self):
+ pass
+
+ def get_url(self):
+ pass
+
+ def step(self, action):
+ pass
+
+ def done(self):
+ pass
diff --git a/webagents_step/environment/liveweb.py b/webagents_step/environment/liveweb.py
new file mode 100644
index 0000000..6680424
--- /dev/null
+++ b/webagents_step/environment/liveweb.py
@@ -0,0 +1,175 @@
+from time import sleep
+import pandas as pd
+import re
+
+from webagents_step.parser import (
+ heihei_web_parser,
+ playwright_parser_nat,
+ playwright_parser_webarena,
+)
+from webagents_step.environment.env import WebEnvironment
+
+
+class LiveWebEnvironmentWrapper(WebEnvironment):
+ def __init__(
+ self,
+ url=None,
+ objective=None,
+ parser_type="heihei",
+ observation_type="text",
+ text_observation_type="accesibility_tree",
+ max_browser_rows=1000,
+ max_steps=50,
+ step_delay=2,
+ headless=False,
+ ):
+ self.url = url
+ self.objective = objective
+ self.headless = headless
+ self.parser_type = parser_type
+ self.observation_type = observation_type
+ self.text_observation_type = text_observation_type
+ self.max_browser_rows = max_browser_rows
+ self.max_steps = max_steps
+
+ self.steps = 0
+ self.is_done = False
+ self.parse_timeout = 5
+ self.step_delay = step_delay
+ self.response = ""
+
+ async def init_parser(self):
+ if self.parser_type == "heihei":
+ self.parser = heihei_web_parser.HeiHeiWebParser()
+ await self.parser.init()
+ elif self.parser_type == "playwright_webarena":
+ self.parser = playwright_parser_webarena.PlaywrightParserWebArena(
+ headless=self.headless,
+ observation_type=self.observation_type,
+ text_observation_type=self.text_observation_type,
+ )
+ self.parser.init()
+ elif self.parser_type == "playwright_nat":
+ self.parser = playwright_parser_nat.PlaywrightParserNat(
+ headless=self.headless
+ )
+ await self.parser.init()
+ else:
+ raise NotImplementedError(f"{self.parser_type} not implemented.")
+
+ if self.url is not None:
+ await self.parser.go_to_page(self.url)
+ self.clear_page_presets()
+ await self.parser.parse_page()
+
+ def clear_page_presets(self):
+ pass
+
+ async def reset(self):
+ await self.close()
+ await self.init_parser()
+
+ async def close(self):
+ await self.parser.close()
+
+ async def observation(self, tab_id=None, format=None):
+ format = self.text_observation_type if format is None else format
+ if self.parser_type == "heihei":
+ try:
+ browser_content = await self.parser.parse_page(
+ format=format, tab_id=tab_id
+ )
+ except:
+ sleep(self.parse_timeout)
+ browser_content = await self.parser.parse_page(
+ format=format, tab_id=tab_id
+ )
+ else:
+ browser_content = await self.parser.parse_page()
+
+ if format not in ["htree", "html", "json"]:
+ browser_content = [str(w) for w in browser_content]
+ browser_content = browser_content[: self.max_browser_rows]
+ browser_content = "\n".join(browser_content)
+
+ return browser_content
+
+ def get_log(self):
+ return self.df_log
+
+ def get_response(self):
+ return self.response
+
+ def get_url(self):
+ return self.parser.get_url()
+
+ async def execute_action(self, action):
+ """
+ Execute a given action based on the action type,
+ - click [id]: Clicks an element based on the provided id.
+ - type [id] [content]: Types the provided content into the element with the specified id.
+ - goto [url]: Navigates to an existing tab at that URL
+ - open [url]: Opens a new tab with provided URL
+ - copy [content]: Copies content, but no-op action
+ - stop [response]: Stops execution and optionally provides a response.
+ """
+ click_match = re.match(r"click \[(\S+)\]", action, re.IGNORECASE)
+ type_match = re.match(r"type \[(\S+)\] \[(.+)\]", action, re.IGNORECASE)
+ goto_match = re.match(r"goto \[(\S+)\]", action, re.IGNORECASE)
+ open_match = re.match(r"open \[(\S+)\]", action, re.IGNORECASE)
+ copy_match = re.match(r"copy \[(\S+)\]", action, re.IGNORECASE)
+ stop_match = re.match(r"stop \[([^\]]*)\]", action, re.IGNORECASE)
+
+ if click_match:
+ id = click_match.group(1)
+ if not id.isdigit():
+ raise Exception("Id not a valid integer")
+ await self.parser.click(int(id))
+
+ elif type_match:
+ id = type_match.group(1)
+ content = type_match.group(2)
+ if not id.isdigit():
+ raise Exception("Id not a valid integer")
+ await self.parser.type(int(id), content)
+
+ elif goto_match:
+ url = goto_match.group(1)
+ tab_id, tab_url = await self.parser.get_tab_from_url(url)
+ await self.parser.go_to_page(url)
+
+ elif open_match:
+ url = open_match.group(1)
+ await self.parser.go_to_page(url)
+
+ elif copy_match:
+ pass
+
+ elif stop_match:
+ self.response = stop_match.group(1)
+ self.is_done = True
+
+ else:
+ print(f"[execute_action] Error {action} not defined")
+
+ async def step(self, action, delay=None):
+ delay = self.step_delay if delay is None else delay
+
+ if self.steps > self.max_steps:
+ print(f"Steps {self.steps} exceeded maximum {self.max_steps}")
+ self.is_done = True
+ return
+
+ print(f"[Step {self.steps+1}] {action}")
+ try:
+ await self.execute_action(action)
+ except Exception as e:
+ print(f"Error while executing action '{action}'. Details: {e}")
+
+ sleep(delay)
+ self.steps = self.steps + 1
+
+ return {"done": self.is_done, "response": self.response}
+
+ def done(self):
+ return self.is_done
diff --git a/webagents_step/parser/playwright_parser_webarena.py b/webagents_step/parser/playwright_parser_webarena.py
new file mode 100644
index 0000000..c5bb5c6
--- /dev/null
+++ b/webagents_step/parser/playwright_parser_webarena.py
@@ -0,0 +1,99 @@
+from sys import platform
+from playwright.sync_api import sync_playwright
+from browser_env.processors import TextObervationProcessor, ImageObservationProcessor
+
+
+class PlaywrightParserWebArena:
+ def __init__(
+ self,
+ headless=True,
+ observation_type="text",
+ text_observation_type="accessibility_tree",
+ viewport_size={"width": 1280, "height": 1080},
+ current_viewport_only=True,
+ ):
+ self.headless = headless
+ self.viewport_size = viewport_size
+ self.current_viewport_only = current_viewport_only
+ self.observation_type = observation_type
+ self.text_observation_type = text_observation_type
+
+ self.playwright = sync_playwright().start()
+ self.browser = self.playwright.chromium.launch(headless=self.headless)
+ self.context = self.browser.new_context(
+ viewport=self.viewport_size,
+ device_scale_factor=1,
+ )
+
+ self.page = self.context.new_page()
+ client = self.page.context.new_cdp_session(self.page)
+ if (self.observation_type == "text") and (
+ self.text_observation_type == "accessibility_tree"
+ ):
+ client.send("Accessibility.enable")
+ self.page.client = client
+
+ ## scratch ##
+ # initialize with html string
+ # self.page.goto(url if "://" in url else "http://" + url)
+ # potentially later
+ # self.page.goto("https://www.google.com", wait_until='networkidle')
+ # print(self.page.accessibility.snapshot())
+ # self.page = self.page.accessibility.snapshot()
+
+ self.text_processor = TextObervationProcessor(
+ observation_type=self.text_observation_type,
+ current_viewport_only=self.current_viewport_only,
+ viewport_size=self.viewport_size,
+ )
+ self.image_processor = ImageObservationProcessor(observation_type="image")
+
+ def clear_page_presets():
+ pass
+
+ def observation_processor(self):
+ if self.observation_type == "text":
+ return self.text_processor
+ elif self.observation_type == "image":
+ return self.image_processor
+ else:
+ raise ValueError("Invalid observation type")
+
+ def get_url(self):
+ return self.page.url
+
+ def go_to_page(self, url: str):
+ self.page.goto(url if "://" in url else "http://" + url)
+
+ def close(self):
+ self.browser.close()
+ self.playwright_context.stop()
+
+ def click_xy(self, x: float, y: float) -> None:
+ viewport_size = self.page.viewport_size
+ self.page.mouse.click(x * viewport_size["width"], y * viewport_size["height"])
+
+ def click(self, id: int) -> None:
+ element_center = self.observation_processor().get_element_center(id)
+ self.click_xy(element_center[0], element_center[1])
+
+ def type(self, id: int, text: str, clear: bool = True):
+ if clear:
+ self.clear(id)
+ self.click(id)
+ self.page.keyboard.type(text)
+
+ def clear(self, id: int) -> None:
+ self.click(id)
+ select_key = "Meta" if platform.startswith("darwin") else "Control"
+ self.page.keyboard.down(select_key)
+ self.page.keyboard.press("a")
+ self.page.keyboard.up(select_key)
+ self.page.keyboard.press("Backspace")
+
+ def parse_page(self):
+ observation = self.observation_processor().process(
+ page=self.page, client=self.page.client
+ )
+
+ return observation
diff --git a/webagents_step/prompts/webarena/step_fewshot_template.py b/webagents_step/prompts/webarena/step_fewshot_template.py
new file mode 100644
index 0000000..a309613
--- /dev/null
+++ b/webagents_step/prompts/webarena/step_fewshot_template.py
@@ -0,0 +1,1457 @@
+github_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_commits [query]`: Given you are in a project page, this subroutine searches Gitlab for commits made to the project and retrieves information about a commit. This function returns the answer to the query.
+`search_issues [query]`: Use this subroutine to find an issue on Gitlab. Any objective that requires finding an issue as an intermediate step, e.g. open latest issue, open issue with and check for X, should call this subroutine
+`create_project [query]`: Given you are in the create new project page, this subroutine completes the act of creating a project, adding members etc.
+`create_group [query]`: Given you are in the create new group page, this subroutine completes the act of creating a group, adding members etc.
+
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+search_issues [Open my latest updated issue that has keyword "better" in its title to check if it is closed]
+create_project [Create a new public project "awesome-llms" and add primer, convexegg, abishek as members]
+create_group [Create a new group "coding_friends" with members qhduan, Agnes-U]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response, e.g. 1 = find_commits [query]
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these GENERAL INSTRUCTIONS:
+* PREVIOUS ACTIONS contains previous actions and subroutine calls with corresponding responses, e.g. 1 = find_commits [query] implies that find_commits subroutine returned a response of 1 commit
+* USE the responses from your subroutine. Do NOT try to solve the subroutine objective again by yourself
+* DO NOT count commits yourself. Return the response from find_commits in PREVIOUS ACTIONS, e.g. 1 = find_commits [query] implies you should return stop [1]
+* If the subroutine returns a response, e.g. Open = search_issues [query], and you have to issue a stop, then issue the same format as that of the response, e.g. stop [Open]
+* If the objective is to check if an issue, pull request, etc is open or closed, respond as though you are answering the question, e.g. "No, it is open", "Yes, it is closed"
+* To access all public projects, you need to navigate to Explore
+* In a repository page, every repository has 4 metrics listed in order Stars, Forks, Merge Requests, and Issues.
+* If a project does not exist, then you cannot do any operations on it like cloning it or creating issues etc.
+* The command to clone a project is git clone [project URL]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the subroutine actions or GENERAL INSTRUCTIONS apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+How many commits did Mike Perotti make to diffusionProject on 03/23/2023?
+OBSERVATION:
+[4] RootWebArea 'Projects · Dashboard · GitLab' focused: True
+ [1664] heading 'The diffusion / diffusionProject.com'
+ [1522] link 'The diffusion / diffusionProject.com
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the dashboard page. The task is to find how many commits Mike Perotti made to the diffusionProject. We have to first click on the project page. Once we are in the project page, we can use the find_commits subroutine. The id corresponding to the project page is 1664. In summary, the next action I will perform is ```click [1664]```
+ACTION:
+click [1664]
+"""},
+{
+"input": """
+OBJECTIVE:
+How many commits did Mike Perotti make to diffusionProject on 03/23/2023?
+OBSERVATION:
+[2234] RootWebArea 'The diffusionProject · GitLab' focused: True
+ [3014] link '0'
+ [3373] link '2,320 Commits'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the diffusionProject page. The task is to find how many commits Mike Perotti made to the diffusionProject. Since we are in the project page, we can use the find_commits subroutine. In summary, the next action I will perform is ```find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]```
+ACTION:
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+"""
+},
+{
+ "input": """
+OBJECTIVE:
+How many stars does a11y-webring.club have?
+OBSERVATION:
+ [1258] link 'A'
+ [1248] heading 'Byte Blaze / a11y-webring.club'
+ [1831] link 'Byte Blaze / a11y-webring.club'
+ [1683] generic 'Public - The project can be accessed without any authentication.'
+ [1673] StaticText 'Owner'
+ [1241] generic 'globe with meridians'
+ [1684] StaticText '🌐'
+ [1771] StaticText ' A webring for digital accessibility practitioners.'
+ [1726] link '2'
+ [1463] link '0'
+ [1325] link '1'
+ [1784] link '4'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Every repository has 4 metrics listed in order Stars, Forks, Merge Requests, and Issues. Hence, [1726] link '2' suggests 2 stars.
+ACTION:
+stop [2]
+"""
+}
+]
+}
+
+find_commits = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+scroll [down]
+stop [Mark made 2 commits on 07/08/2023]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+* To find a list of all commits, you must navigate to the commits section of the repository
+* Look at the first and last date in your observation to know if the desired date is in the range
+* If it's in the range but not visible, that means no commits were made on that date
+* If the date is outside of the range, you need to scroll up/down to get to the desired date range. Scrolling down takes you to a date earlier in time (e.g. Feb 2023 is earlier in time than Mar 2023)
+* To count commits from a specific author, count the number of times their avatar (e.g. img " avatar") appears in the observation.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the general instructions or examples apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+find_commits [How many commits did Mike Perotti make to diffusionProject on 02/02/2023?]
+OBSERVATION:
+[8420] StaticText '02 Feb, 2023'
+ [8423] StaticText '3 commits'
+ [8426] img "Mike Perrotti's avatar"
+ [8428] link 'Coverage improvements (#449)'
+ [8433] link 'Mike Perrotti'
+ [8434] StaticText ' authored '
+ [8435] time 'Feb 1, 2023 10:43pm EST'
+ [8437] StaticText '1 year ago'
+ [10354] button 'Unverified'
+ [8444] StaticText 'a323cbb6'
+ [8442] button 'Copy commit SHA' live: polite atomic: False relevant: additions text
+ [8443] link 'Browse Files'
+ [8455] img "Mike Perrotti's avatar"
+ [8457] link 'updates guidance about numeric table values to be more specific (#451)
+ [8462] link 'Mike Perrotti'
+ [8463] StaticText ' authored '
+ [8464] time 'Feb 1, 2023 10:26pm EST'
+ [8466] StaticText '1 year ago'
+ [10356] button 'Unverified'
+ [8473] StaticText '9bf4ae35'
+ [8471] button 'Copy commit SHA' live: polite atomic: False relevant: additions text
+ [8472] link 'Browse Files'
+ [8484] img "Emily Brick's avatar"
+ [8486] link 'Correct typos on Contributing Guidelines and Follow (#1518)'
+ [8491] link 'Emily Brick'
+ [8492] StaticText ' authored '
+ [8493] time 'Feb 1, 2023 7:40pm EST'
+ [8495] StaticText '1 year ago'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [3373]
+scroll [down]
+scroll [down]
+""",
+"response": """
+REASON:
+The objective is to find how many commits Mike Perotti made on Feb 2, 2023. I see that there has been 3 commits on 02 Feb, 2023.
+However, I must count number of times img "Mike Perrotti's avatar" appears.
+I see [8426] img "Mike Perrotti's avatar", [8455] img "Mike Perrotti's avatar".
+Counting this leads to 2 commits made by Mike Perotti. In summary, the next action I will perform is ```stop [2]```
+ACTION:
+stop [2]
+"""
+},
+]
+}
+
+search_issues = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [Closed]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these general instructions:
+* First navigate the Issues page
+* Once you are in the Issues page, you MUST first navigate to all issues so that you see both open and closed issues for solving the objective
+* You may not see all issues listed at once, use the search bar to search for appropriate keywords and filter down to relevant set of issues
+* If the objective says to "Open ... issue, check if it is X", you must first open the specific issue page by clicking it. Do not stop [] until you have navigated to the specific issue page.
+* Once you are on the issue page, return the appropriate status
+* In your status, if the objective is to check if an issue is open or clossed, respond as though you are answering a question, e.g. "No, it is open", "Yes, it is closed"
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the general instructions or examples apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+Tab 0 (current): Issues \u00b7 Dashboard \u00b7 GitLab\n\n[2234] RootWebArea 'Issues \u00b7 Dashboard \u00b7 GitLab' focused: True\n\t[2241] link 'Skip to content'\n\t[2295] link 'Dashboard'\n\t[4183] button '' hasPopup: menu expanded: False\n\t[3082] textbox 'Search GitLab' required: False\n\t[3025] generic 'Use the shortcut key / to start a search'\n\t[2367] link 'Create new...'\n\t[2369] link 'Issues'\n\t\t[2638] generic '13 assigned issues'\n\t[2370] link 'Merge requests'\n\t\t[2640] generic '8 merge requests'\n\t[2372] link 'To-Do List'\n\t\t[2656] generic 'Todos count'\n\t[2373] link 'Help'\n\t[2375] link 'Byte Blaze'\n\t\t[2690] img 'Byte Blaze'\n\t[2247] main ''\n\t\t[2259] heading 'Issues'\n\t\t[2332] link 'Select project to create issue'\n\t\t[2334] button 'Toggle project select'\n\t\t[2335] link 'Open 13'\n\t\t[2336] link 'Closed 53'\n\t\t[2337] link 'All 66'\n\t\t[2268] link 'Subscribe to RSS feed'\n\t\t[2269] link 'Subscribe to calendar'\n\t\t[2390] button 'Recent searches'\n\t\t[3466] button 'Assignee = Byte Blaze'\n\t\t\t[4251] button ''\n\t\t[3084] textbox '' required: False\n\t\t[3462] button ''\n\t\t[2282] group ''\n\t\t\t[3115] button 'Updated date' hasPopup: listbox\n\t\t\t[2349] link 'Sort direction'\n\t\t[4035] link 'Tm Theme Editor'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. Since the objective is to search over all issues, first I need to navigate to the page containing All issues. Looking at the observation, I am not certain if I am seeing all issues. Hence, I must click the link with text link 'All 66' that suggests that it leads to the page with all issues. In summary, the next action I will perform is ```click [2337]```
+ACTION:
+click [2337]
+"""},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+Tab 0 (current): Issues \u00b7 Dashboard \u00b7 GitLab\n\n[4371] RootWebArea 'Issues \u00b7 Dashboard \u00b7 GitLab' focused: True\n\t[4622] link 'Skip to content'\n\t[4631] link 'Dashboard'\n\t[6161] button '' hasPopup: menu expanded: False\n\t[4383] textbox 'Search GitLab' required: False\n\t[4660] generic 'Use the shortcut key / to start a search'\n\t[4680] link 'Create new...'\n\t[4700] link 'Issues'\n\t\t[4702] generic '13 assigned issues'\n\t[4707] link 'Merge requests'\n\t\t[4710] generic '8 merge requests'\n\t[4732] link 'To-Do List'\n\t\t[4734] generic 'Todos count'\n\t\t\t[4738] StaticText '5'\n\t[4739] link 'Help'\n\t[4780] link 'Byte Blaze'\n\t\t[4782] img 'Byte Blaze'\n\t[4387] main ''\n\t\t[4841] heading 'Issues'\n\t\t[4843] link 'Select project to create issue'\n\t\t[4844] button 'Toggle project select'\n\t\t[4862] link 'Open 13'\n\t\t[4867] link 'Closed 53'\n\t\t[4872] link 'All 66'\n\t\t[4877] link 'Subscribe to RSS feed'\n\t\t[4878] link 'Subscribe to calendar'\n\t\t[4894] button 'Recent searches'\n\t\t[4400] button 'Assignee = Byte Blaze'\n\t\t\t[4946] button ''\n\t\t[4398] textbox '' required: False\n\t\t[4893] button ''\n\t\t[5199] group ''\n\t\t\t[5202] button 'Updated date' hasPopup: listbox\n\t\t\t[5201] link 'Sort direction'\n\t\t[5299] link 'Tm Theme Editor'\n\t\t[5302] StaticText 'byteblaze/a11y-syntax-highlighting#1 '\n\t\t[5303] StaticText '\u00b7 created '\n\t\t[4423] time 'Apr 2, 2020 12:13am EDT'\n\t\t\t[5306] StaticText '3 years ago'\n\t\t[5304] StaticText ' by '\n\t\t[5305] link 'earle'\n\t\t[5312] link 'Assigned to Byte Blaze'\n\t\t[5314] link ' 14'\n\t\t[5320] StaticText 'updated '\n\t\t[4427] time 'Mar 27, 2023 7:15pm EDT'\n\t\t\t[5321]
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+""",
+"response": """
+REASON:
+Let's think step-by-step. I see all issues on the current page. Given the objective, I must search for issues that contain the keyword "homepage content". To do this, I will type the keyword into the search input textbox that appears just before the list of issues. In summary, the next action I will perform is ```type [4398] [homepage content] [1]```
+ACTION:
+type [4398] [homepage content] [1]
+"""
+},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+'\n\t\t[7142] StaticText '\u00b7 created '\n\t\t[6357] time 'Jul 15, 2020 2:48pm EDT'\n\t\t\t[7154] StaticText '3 years ago'\n\t\t[7143] StaticText ' by '\n\t\t[7144] link 'Byte Blaze'\n\t\t[7145] StaticText ' \\xa0 '\n\t\t[7147] link 'bug'\n\t\t[7149] link 'claimed'\n\t\t[7151] link 'redesign'\n\t\t[7153] link 'styling'\n\t\t[7169] StaticText 'CLOSED'\n\t\t[7170] link 'Assigned to Byte Blaze'\n\t\t[7172] link ' 4'\n\t\t[7178] StaticText 'updated '\n\t\t[6361] time 'Mar 23, 2023 4:40am EDT'\n\t\t\t[7179] StaticText '10 months ago'\n\t\t[7182] link \"Rethink the homepage's content\"\n\t\t[7185] StaticText 'a11yproject/a11yproject.com#719 '\n\t\t[7186] StaticText '\u00b7 created '\n\t\t[6367] time 'Dec 30, 2018 2:42pm EST'\n\t\t\t[7197] StaticText '5 years ago'\n\t\t[7187] StaticText ' by '\n\t\t[7188] link 'Byte Blaze'\n\t\t[7190] StaticText '\\xa0 '\n\t\t[7191] link ' Content Updates for 2019'\n\t\t[7192] StaticText ' \\xa0 '\n\t\t[7194] link 'enhancement'\n\t\t[7196] link 'redesign'\n\t\t[7213] StaticText 'CLOSED'\n\t\t[7214] link 'Assigned to Byte Blaze'\n\t\t[7216] link ' 5'\n\t\t[7222] StaticText 'updated '\n\t\t[6371] time 'Mar 23, 2023 4:39am EDT'\n\t\t\t[7223] StaticText
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+type [4398] [homepage content] [1]
+""",
+"response": """
+REASON:
+Let's think step-by-step. I see one issue called "Rethink the homepage's content" that contains the keywords "homepage content". Since it is only one issue, it must be the latest updated issue. According to the general instructions, since the objective asks me to open the issue, I must click the issue. In summary, the next action I will perform is ```click [7182]```
+ACTION:
+click [7182]
+"""
+},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+button 'Collapse sidebar'\n\t[8080] navigation 'Breadcrumbs'\n\t\t[8095] link 'The A11Y Project'\n\t\t[8101] link 'a11yproject.com'\n\t\t[8108] link 'Issues'\n\t\t[8114] link '#719'\n\t[7383] main ''\n\t\t[8138] StaticText 'Closed'\n\t\t[8149] StaticText 'Issue created '\n\t\t[8150] time 'Dec 30, 2018 7:42pm'\n\t\t[8151] StaticText ' by'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+type [4398] [homepage content] [1]
+click [7182]
+""",
+"response": """
+REASON:
+The observation indicates that the issue titled \"Rethink the homepage's content (#719)\" is already open in the current tab. There is also a StaticText element with the text 'Closed' (id 8138), which suggests that the status of the issue is visible on the page. Therefore, no further navigation or clicks are required to complete the task. Hence, I must stop and return the desired response. In summary, the next action I will perform is ```stop [Closed]```
+ACTION:
+"stop [Closed]"
+"""
+},
+]
+}
+
+create_project = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these general instructions:
+1. To add new members, once you have created the project, click on Project Information in the sidebar to be guided to a link with memmbers.
+2. When adding members, first type their name, then click on their name from the down down. Consult PREVIOUS ACTIONS to see if you have typed and selected the names.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+create_project [Create a new public project \"awesome-llms\" and add primer, convexegg, abishek as members]
+OBSERVATION:
+Tab 0 (current): Byte Blaze / awesome-llms \u00b7 GitLab\n\n[34973] RootWebArea 'Byte Blaze / awesome-llms \u00b7 GitLab' focused: True\n\t[35051] link 'Skip to content'\n\t[35060] link 'Dashboard'\n\t[36630] button '' hasPopup: menu expanded: False\n\t[34985] textbox 'Search GitLab' required: False\n\t[35092] generic 'Use the shortcut key / to start a search'\n\t[35112] link 'Create new...'\n\t[35151] link 'Issues'\n\t\t[35153] generic '13 assigned issues'\n\t[35158] link 'Merge requests'\n\t\t[35161] generic '8 merge requests'\n\t[35183] link 'To-Do List'\n\t\t[35185] generic 'Todos count'\n\t[35190] link 'Help'\n\t[35231] link 'Byte Blaze'\n\t\t[35233] img 'Byte Blaze'\n\t[35287] complementary 'Project navigation'\n\t\t[35292] listitem 'awesome-llms'\n\t\t\t[35313] link 'awesome-llms'\n\t\t\t\t[35317] StaticText 'A'\n\t\t[35319] link 'Project information'\n\t\t[35345] link 'Repository'\n\t\t[35387] link 'Issues'\n\t\t\t[35397] StaticText '0'\n\t\t[35421] link 'Merge requests'\n\t\t[35438] link 'CI/CD'\n\t\t[35468] link 'Security & Compliance'\n\t\t[35486] link 'Deployments'\n\t\t[35512] link 'Packages and registries'\n\t\t[35534] link 'Infrastructure'\n\t\t[35560] link 'Monitor'\n\t\t[35590] link 'Analytics'\n\t\t[35616] link 'Wiki'\n\t\t[35629] link 'Snippets'\n\t\t[35642] link 'Settings'\n\t\t[35290] button 'Collapse sidebar'\n\t[4817] generic 'Public - The project can be accessed without any authentication.'\n\t\t[4823] StaticText 'Project ID: 195 '\n\t\t[4824] button 'Copy project ID' live: polite atomic: False relevant: additions text\n\t\t[5115] button '' hasPopup: menu expanded: False\n\t\t[4833] button 'Star'\n\t\t[4834] link '0'\n\t\t[4844] link 'Fork'\n\t\t[4843] link '0'\n\t\t[4857] link '1 Commit'\n\t\t[4865] link '1 Branch'\n\t\t[4873] link '0 Tags'\n\t\t[4881] link '72 KB Project Storage'\n\t\t[6112] link ''\n\t\t[6119] link 'Initial commit'\n\t\t[6122]
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2782]
+type [3910] [awesome-llms] [0]
+click [3903]
+""",
+"response": """
+REASON:
+Let's think step-by-step. The task is to create a new public project named "awesome-llms" and add members to it. However, based on the observation, it appears that the project "awesome-llms" has already been created, as indicated by the presence of elements such as 'Project ID: 195' and 'Initial commit'. The next step is to add members to the project. To do this, I need to navigate to the project information which contains a link to add members. The link to 'Project information' is present in the observation with id 35319. In summary, the next action I will perform is ```click [35319]```
+ACTION:
+click [35319]
+"""},
+]
+}
+
+create_group = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these general instructions:
+1. To add new members, click on the Members tab in the side pane. If you don't see it, click on Group Information in the sidebar to be guided to a link with memmbers.
+2. When adding members, first type their name, then click on their name from the down down. Consult PREVIOUS ACTIONS to see if you have typed and selected the names.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+reddit_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_subreddit [query]`: Find a subreddit corresponding to the query. The query can either be the name of the subreddit or a informative description of what the subreddit may contain. The subroutine hands back control once it navigates to the subreddit by returning "N/A" to denote success.
+`find_user [user_name]`: Navigate to the page of a user with user_name. The page contains all the posts made by the user.
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+find_subreddit [books]
+find_subreddit [something related to driving in Pittsburgh]
+find_subreddit [most appropriate subreddit for X]
+find_user [AdamCannon]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If you have to do a task related to a particular user, first find the user using find_user subroutine
+2. Otherwise, if you have to post or edit a post in a subreddit, first find the subreddit using the find_subreddit subroutine. Pass in as much information in the argument. While find_subreddit will return the most relevant subreddit to your query, it is okay if it does not exactly match your query.
+3. When making a post or a comment to a reply, look at your OBSERVATION or PREVIOUS ACTIONS to make sure you are not repeating the same action.
+4. When typing the "Title" of a submission, make sure to match the phrasing in objective exactly. If the objective said Post "what could X", type that in exactly as the title. In your REASON, you MUST specify the formatting guidelines you are following.
+5. When creating a Forum, be sure to fill in the title, description and sidebar as specified in the objective exactly.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_subreddit = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+* The objective find_subreddit [query] asks you to navigate to the subreddit that best matches the query. The query can be specific or vague.
+* The first step is to navigate to Forums to see the list of subreddits. However, if you have done this already (indicated as non empty PREVIOUS ACTIONS), do not repeat this step.
+* Under forums, you will see only a subset of subreddits. To get the full list of subreddits, you need to navigate to the Alphabetical option.
+* To know you can see the full list of subreddits, you will see 'All Forums' in the observation
+* Often you will not find a focused subreddit that exactly matches your query. In that case, go ahead with the closest relevant subreddit.
+* To know that you have reached a subreddit successfully, you will see '/f/subreddit_name' in the observation.
+* Once you have navigated to any specific subreddit, return stop [N/A]. Even if the subreddit is generally related and not specific to your quwey, stop here and do not try to search again for another subreddit.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_user = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+goto [https://localhost:9999/user/]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_user [user_name] asks you to navigate the page of a user with user_name
+2. To do so, look at the current base URL (e.g. https://localhost:9999) and add a suffix /user/user_name, i.e.
+goto [https://localhost:9999/user/user_name]
+3. Once you have navigated to the user page (as seen in your past actions), return stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+shopping_admin_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_customer_review [query]`: Find customer reviews for a particular product using the query to specify the kind of review.
+`find_order [query]`: Find an order corresponding to a particular customer or order number.
+`search_customer [query]`: Find a customer given some details about them such as their phone number.
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+scroll [down]
+find_customer_review [Show me customer reviews for Zoe products]
+find_order [Most recent pending order by Sarah Miller]
+find_order [Order 305]
+search_customer [Search customer with phone number 8015551212]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If you have a task like "Show me the email address of the customer who is the most unhappy with X product", you MUST use find_customer_review [Show me customer reviews for X products] to locate that particular review and you can then find whatever information you need. Do not try to solve the task without using the subroutine as it contains specific instructions on how to solve it.
+2. If you have a task like "Show me the customers who have expressed dissatisfaction with X product", you MUST use find_customer_review [Show me customer reviews for X product].
+3. If you have a task about a particular order, e.g. "Notify X in their most recent pending order with message Y", you MUST use find_order [Most recent pending order for X] to locate the order, and then do operations on that page. Do this even if the order is visible in the current page.
+4. To write a comment on the order page, you MUST scroll[down] till you find the Comment section. You MUST NOT click on "Comments History" tab, it does not lead you to the right place. Stay on the current page and scroll down to see the comment section.
+5. If you have a task about a particular order, e.g. "Cancel order X", you MUST use find_order [Find order X] to locate the order, and then do operations on that page.
+6. If you have a task like "Find the customer name and email with phone number X", you MUST use search_customer [Search customer with phone number X] to locate the customer, and then answer the query. Do NOT click on CUSTOMERS side panel.
+7. You MUST use Subroutine Actions whenever possible.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the general instructions above apply that would affect the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_customer_review = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_customer_review [query] asks you to navigate to the product page containing customer reviews.
+2. To navigate to a review, first click on REPORTS in the side panel
+3. Once you have clicked on REPORTS, and you see the Reports panel with Marketing, Sales, Reviews, Customers etc, click on By Products under Customers.
+4. Once you are in the Product Reviews Report, you need to locate the product by searching for it. Use the gridcell below Product to search for a product. Do not use other search boxes. Look at the example below where I show you how to search for Zoe in the correct gridcell.
+5. When searching for a product, search the first word only like Zoe, or Antonia or Chloe.
+6. Once the product shows up, click on 'Show Reviews'.
+7. Once all the reviews show up, return stop [N/A] to hand back control to the agent that queried you.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+ {
+"input": """
+OBJECTIVE:
+find_product_review [Show me the review of the customer who is the most unhappy with the style of Zoe products]
+OBSERVATION:
+Tab 0 (current): Product Reviews Report / Reviews / Reports / Magento Admin\n\t\t[1992] table ''\n\t\t\t[2723] row ''\n\t\t\t\t[2724] columnheader 'ID' required: False\n\t\t\t\t[2725] columnheader 'Product' required: False\n\t\t\t\t[2726] columnheader '\u2191 Reviews' required: False\n\t\t\t\t[2727] columnheader 'Average' required: False\n\t\t\t\t[2728] columnheader 'Average (Approved)' required: False\n\t\t\t\t[2729] columnheader 'Last Review' required: False\n\t\t\t\t[2730] columnheader 'Action' required: False\n\t\t\t[1994] row ''\n\t\t\t\t[1995] gridcell '' required: False\n\t\t\t\t\t[1996] textbox '' required: False\n\t\t\t\t[1997] gridcell '' required: False\n\t\t\t\t\t[1998] textbox '' required: False\n\t\t\t\t[1999] gridcell '' required: False\n\t\t\t\t\t[2000] textbox '' required: False\n\t\t\t\t[2001] gridcell '' required: False\n\t\t\t\t\t[2002] textbox '' required: False\n\t\t\t\t[2003] gridcell '' required: False\n\t\t\t\t\t[2004] textbox '' required: False\n\t\t\t\t[2005] gridcell 'undefined \\ue627 undefined \\ue627' required: False\n\t\t\t\t\t[2008] textbox 'From' required: False\n\t\t\t\t\t[3196] button 'undefined \\ue627'\n\t\t\t\t\t[2010] textbox 'To' required: False\n\t\t\t\t\t[3201] button 'undefined \\ue627'\n\t\t\t\t[2747] gridcell '\\xa0' required: False\n\t\t\t[2759] row 'https://localhost:7780/admin/review/product/index/productId/1620/'\n\t\t\t\t[2779] gridcell '1620' required: False\n\t\t\t\t[2780] gridcell 'Erica Evercool Sports Bra' required: False\n\t\t\t\t[2781] gridcell '4' required: False\n\t\t\t\t[2782] gridcell '60.0000' required: False\n\t\t\t\t[2783] gridcell '60.0000' required: False\n\t\t\t\t[2784] gridcell 'Apr 19, 2023, 12:15:19 PM' required: False\n\t\t\t\t[2785] gridcell 'Show Reviews' required: False\n\t\t\t\t\t[2792] link 'Show Reviews'\n\t\t\t[2760] row 'https://localhost:7780/admin/review/product/index/productId/39/'\n\t\t\t\t[2794] gridcell '39' required: False\n\t\t\t\t[2795] gridcell
+URL:
+https://localhost:7780/admin/reports/report_review/product/
+PREVIOUS ACTIONS:
+click [1195]
+click [1531]
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the Product Review Report page. We have to search for a Zoe product. To do so, I have to type in Zoe in a gridcell corresponding to the Product column. The Product column is the second column after ID. Therefore I have to find the second gridcell just below it. The first one is 1995, and the second one is 1997. In summary, the next action I will perform is type [1997] [Zoe] [1]
+ACTION:
+type [1997] [Zoe] [1]
+"""},
+]
+}
+
+find_order = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+scroll [down]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+* The objective find_order [query] asks you to navigate to the order page corresponding to the query
+* To navigate to orders, first go to SALES in the side panel
+* Once you have clicked on SALES, go to Orders
+* Once you are in the orders page, you have to use the 'Filter' button to filter down to desired criteria
+* Desired criterias include filtering down to a specific order ID field or Name field. ONLY use fields that are in the objective
+* You MUST use Filter to find orders instead of using the search bar
+* If there are any active filters, be sure to clear them before entering your filter criteria
+* In your filtered list of orders, if you don't find the desired order, make sure to scroll down till you find the order or reach end of page (typically indicated by 'Copyright © ...' in the observation)
+* Once you have found the order, go to View to open the order
+* Once you are in the desired order page (as noted by "Order & Account Information") you MUST return stop [N/A] to hand back control to the agent that queried you. Do not go back to another page.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_customer = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective search_customer [query] asks you to search for customer details corresponding to the query
+2. To navigate to customers, first click on CUSTOMERS in the side panel
+3. Once you have clicked on CUSTOMERS, click on All Customers.
+4. Once you are in the customers page, you have to use the 'Search by keyword' text box to search for your customer. Always be sure to search first. For example, for find_order [Search customer with phone number 8015551212], search 8015551212.
+5. If the page shows a number has already been searched, click on Clear All first. Then proceed with the search.
+6. Once you are done with the search, and the customer with matching query shows up, you MUST return stop [N/A] to hand back control to the agent that queried you. Do not go back to another page.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+shopping_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+`hover [id]`: Hover over an element with id.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`search_order [question]`: Search orders to answer a question about my orders
+`find_products [query]`: Find products that match a query
+`search_reviews [query]`: Search reviews to answer a question about reviews
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+scroll [down]
+hover [11]
+search_order [How much I spend on 4/19/2023 on shopping at One Stop Market?]
+list_products [List products from PS4 accessories category by ascending price]
+search_reviews [List out reviewers, if exist, who mention about ear cups being small]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these GENERAL INSTRUCTIONS:
+* If the OBJECTIVE is a question about my orders, you MUST use search_order [question] to answer the question e.g. How much did I spend on X, or What is the size of X that I bought, or Change the delivery address for X.
+Do not try to solve the task without using search_order as it contains specific instructions on how to solve it. Do not click on MyAccount directly.
+* The response from subroutines is stored in PREVIOUS ACTIONS. For example, $0 = search_order [How much I spend on X?] means that the response was $0. In that case, return the answer directly, e.g. stop [$0]. If the response was N/A, reply stop [N/A]. Trust the answer returned by search_order.
+* If the OBJECTIVE is a question about listing / showing products, you MUST use list_products. For example,
+list_products [List products from X]
+list_products [Show me the most expensive product from X]
+* If the OBJECTIVE requires you to retrieve details about a particular order you placed liked SKU, you MUST first use search_order [] to retrieve the SKU.
+For example, if the OBJECTIVE is "Fill the form for a refund on X .... Also, ensure to include the order number #161 and the product SKU.", you must first issue search_order [Give me the SKU of X from order number #161].
+* If the OBJECTIVE requires order id and amount, you must first issue search_order [Give me the order id and the amount for X]
+* If the OBJECTIVE is about reviews for the product, you MUST use search_reviews. For example, search_reviews [List out reviewers ..] or search_reviews [What are the main criticisms of X]
+* Return the response from search_reviews VERBATIM. Trust that it has solved the OBJECTIVE correctly.
+* When filling out a form for refund, you must mention the word refund. Also, you MUST NOT use the word "just" or "which". This is against formatting guidelines. E.g. say "It broke after three days" rather than "which broke after just three days" or "The product broke after three days".
+* The Contact Us link is usually at the bottom of a page, scroll down to find it.
+* If the OBJECTIVE asks you to "Draft" something, perform all necessary actions except submitting at the end. Do NOT submit as this is a draft.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the subroutine actions or GENERAL INSTRUCTIONS apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_order = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`note [content]`: Use this to make a personal note of some content you would like to remember. This shows up in your history of previous actions so you can refer to it.
+`go_back`: Navigate to the previously viewed page.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+note [Spent $10 on 4/1/2024]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these GENERAL INSTRUCTIONS:
+* Navigate to My Account, then My Orders to access all orders.
+* The orders are sorted by descending date. Click on Page Next to go to a earlier date. Click on Page Previous to go to a earlier date.
+If you don't see an order for a date, and the first order on the page is after the date, the last order on the page is before the date, then it means there is no order for the date.
+* In your REASON, state what is the current range, what range you are looking for, and whether you should search for an earlier or a later date.
+* If you have to find the total amount you spent on orders that span multiple pages, use note [Spent $10 on 4/1/2024] to make a personal note before moving on to the next page. When you are done, you can look at PREVIOUS ACTIONS to find all notes.
+* When you are adding numbers, work out each addition step by step in REASON.
+* Use go_back to go back to a previous page from an order. But before you do, use note [] to make a note that you checked the page, e.g. note [Checked order on 11/29/2023, no picture frame.]
+* If you are in an order page and need to go back, issue go_back. Don't click on My Orders else you have to start from all over again.
+* Do not keep visiting the same order page over and over again!
+To prevent this, whenever you visit a page, always make a note. For example note [Nothing relevant purchased on September 29, 2022]
+See note [] to see what dates you have visit, and be sure to not visit that page again.
+* Once you are done visiting all the pages, return stop [answer] with the answer to the query.
+* If the question is how much did I spend on a date, and I didn't spend anything, return stop [$0]
+* If the status of an order shows cancelled, that means I did not spend that money
+* If you are asked to change the delivery address on an order, you can't. Reply stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the GENERAL INSTRUCTIONS apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+list_products = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id. Use this whenever any element has the field hasPopup: menu
+`goto [url]`: Navigate to a specific URL. Use this when needing to sort by price. Refer to instructions below.
+`note [content]`: Use this to make a personal note of some content you would like to remember. This shows up in your history of previous actions so you can refer to it.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+hover [77]
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. To find a product category, you MUST use hover [id] to expand the popup Menu till you find the leaf element that has no popup. Then click [id] on the leaf element.
+For exmaple, to find PS 4 accessories you must hover over Video Games, then hover over Playstation 4, then click on Accessories.
+Use note [] eveytime you hover on an item, and don't find the category. This is to ensure you don't keep trying the same category repeatedly.
+2. To sort current list of products by price and in ascending order, you MUST use the goto [url] action by appending ?product_list_order=price to the current URL. For example:
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+then issue goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price]
+3. To sort in descending order, you MUST use the goto [url] action by appending ?product_list_order=price&product_list_dir=desc, e.g.
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price&product_list_dir=desc]
+4. To list all items less than a particular price, e.g. $25, you MUST use the goto [url] action by appending ?price=0-25
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?price=0-25]
+5. Once you are done in stop [N/A]
+6. If the OBJECTIVE asks you to show the most expensive product, you must click on the product.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_reviews = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+`note [content]`: Use this to make a personal note of some content you would like to remember. This shows up in your history of previous actions so you can refer to it.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+scroll [down]
+note [Reviewer X made comment Y]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+* If you are not in the product page, you can search for the product using the search bar.
+* To find the list of reviews, search for a link with Reviewers. If you can't find it, scroll down to search for it.
+* Iterate over all reviews. For every relevant review, make a note [reviewer_name: review_info]. Record the relevant reviewer_name and review VERBATIM. Once you are done with all the reviews in a page, scroll down to access more reviews.
+* Refer to PREVIOUS ACTIONS to know which reviews you have noted already. If you have noted a review already, look for the next review in your current OBSERVATION or scroll down.
+* Do NOT repeat the note [] action for the same review.
+* Not all reviews will be visible on the reviews page. You MUST scroll down till you reach the end of the page. You will know that you have reached the end of the page if you see “Contact Us” in the OBSERVATION.
+* Once you have scrolled through all reviews, combine all your noted reviews that you can find under PREVIOUS ACTIONS. To combine, create a list of dicts where every dict has a name and review key. Be sure to capture ALL the reviews in your note. Return that as stop [{name: reviewer_name_1, review: review_1}, {name: reviewer_name_2, review: review_2}, ..]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+In your REASON, you MUST specify if any of the general instructions apply and how that affects the action you choose.
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+maps_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_directions [query]`: Find directions between two locations to answer the query
+`search_nearest_place [query]`: Find places near a given location
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+scroll [down]
+find_directions [Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University]
+search_nearest_place [Tell me the closest cafe(s) to CMU Hunt library]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If the OBJECTIVE is about finding directions from A to B, you MUST use find_directions [] subroutine.
+e.g. find_directions [Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University]
+2. If the OBJECTIVE is about searching nearest place to a location, you MUST use search_nearest_place [] subroutine.
+e.g. search_nearest_place [Tell me the closest restaurant(s) to Cohon University Center at Carnegie Mellon University]
+3. If the OBJECTIVE is to pull up a description, once that place appears in the sidepane, return stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_directions = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [5h 47min]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. First click on "Find directions between two points", then enter From and To Fields, and click search.
+2. If you have to find directions to social security administration in Pittsburgh, search for it in a structured format like Social Security Administration, Pittsburgh.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_nearest_place = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [De Fer Coffee & Tea, La Prima Espresso, Rothberg's Roasters II, Cafe Phipps, La Prima Espresso, Starbucks]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. For searches that refer to CMU, e.g. "find cafes near CMU Hunt Library"
+a. You have to first center your map around a location. If you have to find cafes near CMU Hunt Library, the first step is to make sure the map is centered around Carnegie Mellon University. To do that, first search for Carnegie Mellon University and then click [] on a list of location that appears. You MUST click on the Carnegie Mellon University location to center the map. Else the map will not centered. E.g click [646]
+b. Now that your map is centered around Carnegie Mellon University, directly search for "cafes near Hunt Library". Do not include the word CMU in the search item.
+The word CMU cannot be parsed by maps and will result in an invalid search.
+c. When your search returns a list of elements, return them in a structured format like stop [A, B, C]
+2. For searches that don't refer to CMU
+a. No need to center the map. Directly search what is specified in OBJECTIVE, e.g. "bars near Carnegie Music Hall"
+b. When your search returns a list of elements, return them in a structured format like stop [A, B, C]
+3. Be sure to double check whether the OBJECTIVE has CMU or not and then choose between instruction 1 and 2.
+4. Remember that the word CMU cannot be typed in the search bar as it cannot be parsed by maps.
+5. Remember that if you want to center your map around Carnegie Mellon University, you have to click on it after you search for it. Check your PREVIOUS ACTIONS to confirm you have done so, e.g. click [646] should be in the previous actions.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
\ No newline at end of file
diff --git a/webagents_step/prompts/webarena/step_fewshot_template_adapted.py b/webagents_step/prompts/webarena/step_fewshot_template_adapted.py
new file mode 100644
index 0000000..df40fcb
--- /dev/null
+++ b/webagents_step/prompts/webarena/step_fewshot_template_adapted.py
@@ -0,0 +1,1396 @@
+github_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_commits [query]`: Given you are in a project page, this subroutine searches Gitlab for commits made to the project and retrieves information about a commit. This function returns the answer to the query.
+`search_issues [query]`: Given you are in my issue page, this subroutine searches Gitlab to find issue that matches the query. Any objective that says "openn my latest issue" or "open issue with in the title" must be passed through this subroutine.
+`create_project [query]`: Given you are in the create new project page, this subroutine completes the act of creating a project, adding members etc.
+`create_group [query]`: Given you are in the create new group page, this subroutine completes the act of creating a group, adding members etc.
+
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+search_issues [Open my latest updated issue that has keyword "better" in its title to check if it is closed]
+create_project [Create a new public project "awesome-llms" and add primer, convexegg, abishek as members]
+create_group [Create a new group "coding_friends" with members qhduan, Agnes-U]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response, e.g. 1 = find_commits [query]
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+
+* Use responses from your subroutine calls to answer the objective. The PREVIOUS ACTIONS section lists all previous actions and their corresponding responses. For instance, an entry of 1 = find_commits [query] implies that the find_commits subroutine you invoked returned a response of 1. Use that response to solve the objective.
+* If the subroutine returns a response, e.g. Open = search_issues [query], and you have to issue a stop, then issue the same format as that of the response, e.g. stop [Open]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+How many commits did Mike Perotti make to diffusionProject on 03/23/2023?
+OBSERVATION:
+RootWebArea 'Projects · Dashboard · GitLab' focused: True
+ heading 'The diffusion / diffusionProject.com'
+ link [1522] 'The diffusion / diffusionProject.com
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the dashboard page. The task is to find how many commits Mike Perotti made to the diffusionProject. We have to first click on the project page. Once we are in the project page, we can use the find_commits subroutine. The id corresponding to the project page is 1522. In summary, the next action I will perform is ```click [1522]```
+ACTION:
+click [1522]
+"""},
+{
+"input": """
+OBJECTIVE:
+How many commits did Mike Perotti make to diffusionProject on 03/23/2023?
+OBSERVATION:
+RootWebArea 'The diffusionProject · GitLab' focused: True
+ link [3014] '0'
+ link [3373] '2,320 Commits'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the diffusionProject page. The task is to find how many commits Mike Perotti made to the diffusionProject. Since we are in the project page, we can use the find_commits subroutine. In summary, the next action I will perform is ```find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]```
+ACTION:
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+"""
+},
+]
+}
+
+find_commits = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+stop [Mark made 2 commits on 07/08/2023]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+OBSERVATION:
+RootWebArea 'The diffusionProject · GitLab' focused: True
+ link [3014] '0'
+ link [3373] '2,320 Commits'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. To find the number of commits made by Mike Perotti on a specific date, I need to navigate to the commits section of the repository. The link with the text '2,320 Commits' suggests that it leads to the commit history. Therefore, I will click on this link to proceed to the commits section. In summary, the next action I will perform is ```click [3373]```
+ACTION:
+click [3373]
+"""},
+{
+"input": """
+OBJECTIVE:
+find_commits [How many commits did Mike Perotti make to diffusionProject on 03/23/2023?]
+OBSERVATION:
+text '23 Mar, 2023'
+text '3 commits'
+link [11366] "Emily Brick's avatar"
+link [11369] 'Coverage improvements (#449)'
+button [11371] 'Toggle commit description'
+link [11380] 'Emily Brick'
+time 'Mar 23, 2023 7:58pm EDT'
+link [11440] 'Browse Files'
+link [11451] "Mike Perrotti's avatar"
+link [11454] 'updates guidance about numeric table values to be more specific (#451)'
+link [11459] 'Mike Perrotti'
+text ' authored '
+time 'Mar 23, 2023 2:58pm EDT'
+button [13266] 'Unverified'
+text 'da9575e5'
+link [11469] 'Browse Files'
+link [11480] "Cole Bemis's avatar"
+link [11483] 'Add SSR compatibility to component lifecycle criteria (#440)'
+link [11494] 'Cole Bemis'
+time 'Mar 22, 2023 2:40pm EDT'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [3373]
+""",
+"response": """
+REASON:
+Let's think step-by-step. I have already clicked on the link for commits.
+From the browser content, I can see this is a list of commits for the diffusionProject.
+I see that there has been 3 commits on 03/23/2023. I see that Mike Perrotti made 1 commit on Mar 23, 2023 2:58pm EDT. In summary, the next action I will perform is ```stop [1]```
+ACTION:
+stop[1]
+"""
+},
+]
+}
+
+search_issues = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [Closed]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+
+Please follow these general instructions:
+1. By default you begin with the page containing all open issues. If the objective requires you to search over all issues, e.g. "Open my latest updated issue ... check if closed", make sure that you navigate to the page containing "all issues"".
+2. If the objective says "Open ... issue to check if it is closed", this means:
+a. First open the issue being referred to by clicking on it
+b. Then return the status, i.e. stop [open], stop [closed].
+Do not return stop [] until you are sure that you have clicked on the issue.
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+Tab 0 (current): Issues \u00b7 Dashboard \u00b7 GitLab\n\nRootWebArea 'Issues \u00b7 Dashboard \u00b7 GitLab' focused: True\n\tlink [2241] 'Skip to content'\n\tlink [2295] 'Dashboard'\n\tbutton [4183] '' hasPopup: menu expanded: False\n\ttextbox [3082] 'Search GitLab' required: False\n\tgeneric [3025] 'Use the shortcut key / to start a search'\n\tlink [2367] 'Create new...'\n\tlink [2369] 'Issues'\n\t\tgeneric [2638] '13 assigned issues'\n\tlink [2370] 'Merge requests'\n\t\tgeneric [2640] '8 merge requests'\n\tlink [2372] 'To-Do List'\n\t\tgeneric [2656] 'Todos count'\n\tlink [2373] 'Help'\n\tlink [2375] 'Byte Blaze'\n\t\timg 'Byte Blaze'\n\tmain [2247] ''\n\t\theading 'Issues'\n\t\tlink [2332] 'Select project to create issue'\n\t\tbutton [2334] 'Toggle project select'\n\t\tlink [2335] 'Open 13'\n\t\tlink [2336] 'Closed 53'\n\t\tlink [2337] 'All 66'\n\t\tlink [2268] 'Subscribe to RSS feed'\n\t\tlink [2269] 'Subscribe to calendar'\n\t\tbutton [2390] 'Recent searches'\n\t\tbutton [3466] 'Assignee = Byte Blaze'\n\t\t\tbutton [4251] ''\n\t\ttextbox [3084] '' required: False\n\t\tbutton [3462] ''\n\t\tgroup [2282] ''\n\t\t\tbutton [3115] 'Updated date' hasPopup: listbox\n\t\t\tlink [2349] 'Sort direction'\n\t\tlink [4035] 'Tm Theme Editor'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+
+""",
+"response": """
+REASON:
+Let's think step-by-step. Since the objective is to search over all issues, first I need to navigate to the page containing All issues. Looking at the observation, I am not certain if I am seeing all issues. Hence, I must click the link with text link 'All 66' that suggests that it leads to the page with all issues. In summary, the next action I will perform is ```click [2337]```
+ACTION:
+click [2337]
+"""},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+Tab 0 (current): Issues \u00b7 Dashboard \u00b7 GitLab\n\nRootWebArea 'Issues \u00b7 Dashboard \u00b7 GitLab' focused: True\n\tlink [4622] 'Skip to content'\n\tlink [4631] 'Dashboard'\n\tbutton [6161] '' hasPopup: menu expanded: False\n\ttextbox [4383] 'Search GitLab' required: False\n\tgeneric [4660] 'Use the shortcut key / to start a search'\n\tlink [4680] 'Create new...'\n\tlink [4700] 'Issues'\n\t\tgeneric [4702] '13 assigned issues'\n\tlink [4707] 'Merge requests'\n\t\tgeneric [4710] '8 merge requests'\n\tlink [4732] 'To-Do List'\n\t\tgeneric [4734] 'Todos count'\n\t\t\ttext '5'\n\tlink [4739] 'Help'\n\tlink [4780] 'Byte Blaze'\n\t\timg 'Byte Blaze'\n\tmain [4387] ''\n\t\theading 'Issues'\n\t\tlink [4843] 'Select project to create issue'\n\t\tbutton [4844] 'Toggle project select'\n\t\tlink [4862] 'Open 13'\n\t\tlink [4867] 'Closed 53'\n\t\tlink [4872] 'All 66'\n\t\tlink [4877] 'Subscribe to RSS feed'\n\t\tlink [4878] 'Subscribe to calendar'\n\t\tbutton [4894] 'Recent searches'\n\t\tbutton [4400] 'Assignee = Byte Blaze'\n\t\t\tbutton [4946] ''\n\t\ttextbox [4398] '' required: False\n\t\tbutton [4893] ''\n\t\tgroup [5199] ''\n\t\t\tbutton [5202] 'Updated date' hasPopup: listbox\n\t\t\tlink [5201] 'Sort direction'\n\t\tlink [5299] 'Tm Theme Editor'\n\t\ttext 'byteblaze/a11y-syntax-highlighting#1 '\n\t\ttext '\u00b7 created '\n\t\ttime 'Apr 2, 2020 12:13am EDT'\n\t\t\ttext '3 years ago'\n\t\ttext ' by '\n\t\tlink [5305] 'earle'\n\t\tlink [5312] 'Assigned to Byte Blaze'\n\t\tlink [5314] ' 14'\n\t\ttext 'updated '\n\t\ttime 'Mar 27, 2023 7:15pm EDT'\n\t\t\t[5321]
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+""",
+"response": """
+REASON:
+Let's think step-by-step. I see all issues on the current page. Given the objective, I must search for issues that contain the keyword "homepage content". To do this, I will type the keyword into the search input textbox that appears just before the list of issues. In summary, the next action I will perform is ```type [4398] [homepage content] [1]```
+ACTION:
+type [4398] [homepage content] [1]
+"""
+},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+'\n\t\ttext '\u00b7 created '\n\t\ttime 'Jul 15, 2020 2:48pm EDT'\n\t\t\ttext '3 years ago'\n\t\ttext ' by '\n\t\tlink [7144] 'Byte Blaze'\n\t\ttext ' \\xa0 '\n\t\tlink [7147] 'bug'\n\t\tlink [7149] 'claimed'\n\t\tlink [7151] 'redesign'\n\t\tlink [7153] 'styling'\n\t\ttext 'CLOSED'\n\t\tlink [7170] 'Assigned to Byte Blaze'\n\t\tlink [7172] ' 4'\n\t\ttext 'updated '\n\t\ttime 'Mar 23, 2023 4:40am EDT'\n\t\t\ttext '10 months ago'\n\t\tlink [7182] \"Rethink the homepage's content\"\n\t\ttext 'a11yproject/a11yproject.com#719 '\n\t\ttext '\u00b7 created '\n\t\ttime 'Dec 30, 2018 2:42pm EST'\n\t\t\ttext '5 years ago'\n\t\ttext ' by '\n\t\tlink [7188] 'Byte Blaze'\n\t\ttext '\\xa0 '\n\t\tlink [7191] ' Content Updates for 2019'\n\t\ttext ' \\xa0 '\n\t\tlink [7194] 'enhancement'\n\t\tlink [7196] 'redesign'\n\t\ttext 'CLOSED'\n\t\tlink [7214] 'Assigned to Byte Blaze'\n\t\tlink [7216] ' 5'\n\t\ttext 'updated '\n\t\ttime 'Mar 23, 2023 4:39am EDT'\n\t\t\ttext
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+type [4398] [homepage content] [1]
+""",
+"response": """
+REASON:
+Let's think step-by-step. I see one issue called "Rethink the homepage's content" that contains the keywords "homepage content". Since it is only one issue, it must be the latest updated issue. According to the general instructions, since the objective asks me to open the issue, I must click the issue. In summary, the next action I will perform is ```click [7182]```
+ACTION:
+click [7182]
+"""
+},
+{
+"input": """
+OBJECTIVE:
+search_issues [Open my latest updated issue that has keyword \"homepage content\" in its title to check if it is closed]
+OBSERVATION:
+button 'Collapse sidebar'\n\t[8080] navigation 'Breadcrumbs'\n\t\tlink [8095] 'The A11Y Project'\n\t\tlink [8101] 'a11yproject.com'\n\t\tlink [8108] 'Issues'\n\t\tlink [8114] '#719'\n\tmain [7383] ''\n\t\ttext 'Closed'\n\t\ttext 'Issue created '\n\t\ttime 'Dec 30, 2018 7:42pm'\n\t\ttext ' by'
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2337]
+type [4398] [homepage content] [1]
+click [7182]
+""",
+"response": """
+REASON:
+The observation indicates that the issue titled \"Rethink the homepage's content (#719)\" is already open in the current tab. There is also a text element with the text 'Closed' (id 8138), which suggests that the status of the issue is visible on the page. Therefore, no further navigation or clicks are required to complete the task. Hence, I must stop and return the desired response. In summary, the next action I will perform is ```stop [Closed]```
+ACTION:
+"stop [Closed]"
+"""
+},
+]
+}
+
+create_project = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these general instructions:
+1. To add new members, once you have created the project, click on Project Information in the sidebar to be guided to a link with memmbers.
+2. When adding members, first type their name, then click on their name from the down down. Consult PREVIOUS ACTIONS to see if you have typed and selected the names.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+{
+"input": """
+OBJECTIVE:
+create_project [Create a new public project \"awesome-llms\" and add primer, convexegg, abishek as members]
+OBSERVATION:
+Tab 0 (current): Byte Blaze / awesome-llms \u00b7 GitLab\n\nRootWebArea 'Byte Blaze / awesome-llms \u00b7 GitLab' focused: True\n\tlink [35051] 'Skip to content'\n\tlink [35060] 'Dashboard'\n\tbutton [36630] '' hasPopup: menu expanded: False\n\ttextbox [34985] 'Search GitLab' required: False\n\tgeneric [35092] 'Use the shortcut key / to start a search'\n\tlink [35112] 'Create new...'\n\tlink [35151] 'Issues'\n\t\tgeneric [35153] '13 assigned issues'\n\tlink [35158] 'Merge requests'\n\t\tgeneric [35161] '8 merge requests'\n\tlink [35183] 'To-Do List'\n\t\tgeneric [35185] 'Todos count'\n\tlink [35190] 'Help'\n\tlink [35231] 'Byte Blaze'\n\t\timg 'Byte Blaze'\n\tcomplementary [35287] 'Project navigation'\n\t\tlistitem 'awesome-llms'\n\t\t\tlink [35313] 'awesome-llms'\n\t\t\t\ttext 'A'\n\t\tlink [35319] 'Project information'\n\t\tlink [35345] 'Repository'\n\t\tlink [35387] 'Issues'\n\t\t\ttext '0'\n\t\tlink [35421] 'Merge requests'\n\t\tlink [35438] 'CI/CD'\n\t\tlink [35468] 'Security & Compliance'\n\t\tlink [35486] 'Deployments'\n\t\tlink [35512] 'Packages and registries'\n\t\tlink [35534] 'Infrastructure'\n\t\tlink [35560] 'Monitor'\n\t\tlink [35590] 'Analytics'\n\t\tlink [35616] 'Wiki'\n\t\tlink [35629] 'Snippets'\n\t\tlink [35642] 'Settings'\n\t\tbutton [35290] 'Collapse sidebar'\n\tgeneric [4817] 'Public - The project can be accessed without any authentication.'\n\t\ttext 'Project ID: 195 '\n\t\tbutton [4824] 'Copy project ID' live: polite atomic: False relevant: additions text\n\t\tbutton [5115] '' hasPopup: menu expanded: False\n\t\tbutton [4833] 'Star'\n\t\tlink [4834] '0'\n\t\tlink [4844] 'Fork'\n\t\tlink [4843] '0'\n\t\tlink [4857] '1 Commit'\n\t\tlink [4865] '1 Branch'\n\t\tlink [4873] '0 Tags'\n\t\tlink [4881] '72 KB Project Storage'\n\t\tlink [6112] ''\n\t\tlink [6119] 'Initial commit'\n\t\t[6122]
+URL:
+https://localhost:8023/
+PREVIOUS ACTIONS:
+click [2782]
+type [3910] [awesome-llms] [0]
+click [3903]
+""",
+"response": """
+REASON:
+Let's think step-by-step. The task is to create a new public project named "awesome-llms" and add members to it. However, based on the observation, it appears that the project "awesome-llms" has already been created, as indicated by the presence of elements such as 'Project ID: 195' and 'Initial commit'. The next step is to add members to the project. To do this, I need to navigate to the project information which contains a link to add members. The link to 'Project information' is present in the observation with id 35319. In summary, the next action I will perform is ```click [35319]```
+ACTION:
+click [35319]
+"""},
+]
+}
+
+create_group = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these general instructions:
+1. To add new members, click on the Members tab in the side pane. If you don't see it, click on Group Information in the sidebar to be guided to a link with memmbers.
+2. When adding members, first type their name, then click on their name from the down down. Consult PREVIOUS ACTIONS to see if you have typed and selected the names.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+reddit_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_subreddit [query]`: Find a subreddit corresponding to the query. The query can either be the name of the subreddit or a vague description of what the subreddit may contain. The subroutine hands back control once it navigates to the subreddit.
+`find_user [user_name]`: Navigate to the page of a user with user_name. The page contains all the posts made by the user.
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+find_subreddit [books]
+find_subreddit [something related to driving in Pittsburgh]
+find_user [AdamCannon]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If you have to do a task related to a particular user, first find the user using find_user subroutine
+2. Otherwise, if you have to post or edit a post in a subreddit, first find the subreddit using the find_subreddit subroutine
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_subreddit = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_subreddit [query] asks you to navigate to the subreddit that best matches the query. The query can be specific or vague.
+2. To navigate to a subreddit, first click on Forums from the top menu.
+3. Once you are in the Forums page, and you see the Alphabetical option, click on it to see a list of all subreddits alphabetically.
+4. Once you are in the page with all the subreddits listed alphabetically, click on the subreddit that matches the query
+5. Once you have navigated to the subreddit, return stop [N/A]. You can check that you are in the subreddit by looking at the current observation and seeing "heading '/f/subreddit_name'". You will also see a number of posts. If the subreddit_name vaguely matches the query, it means you are already in the subreddit and should stop, e.g. gaming and games are the same subreddit.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_user = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [issue] [1]
+stop [N/A]
+goto [http://localhost:9999/user/]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_user [user_name] asks you to navigate the page of a user with user_name
+2. To do so, look at the current base URL (e.g. http://localhost:9999) and add a suffix /user/user_name, i.e.
+goto [http://localhost:9999/user/user_name]
+3. Once you have navigated to the user page (as seen in your past actions), return stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+shopping_admin_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_customer_review [query]`: Find customer reviews for a particular product using the query to specify the kind of review.
+`find_order [query]`: Find an order corresponding to a particular customer or order number.
+`search_customer [query]`: Find a customer given some details about them such as their phone number.
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+find_customer_review [Show me customer reviews for Zoe products]
+find_order [Most recent pending order by Sarah Miller]
+find_order [Order 305]
+search_customer [Search customer with phone number 8015551212]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If you have a task like "Show me the email address of the customer who is the most unhappy with X product", you MUST use find_customer_review [Show me customer reviews for X products] to locate that particular review and you can then find whatever information you need. Do not try to solve the task without using the subroutine as it contains specific instructions on how to solve it.
+2. If you have a task like "Show me the customers who have expressed dissatisfaction with X product", you MUST use find_customer_review [Show me customer reviews for X product].
+3. If you have a task about a particular order, e.g. "Notify X in their most recent pending order with message Y", you MUST use find_order [Most recent pending order for X] to locate the order, and then do operations on that page.
+4. To write a comment on the order page, you MUST NOT click on "Comments History" tab, it does not lead you to the right place. Stay on the current page and check the comment section.
+5. If you have a task about a particular order, e.g. "Cancel order 305", you MUST use find_order [Find order 305] to locate the order, and then do operations on that page.
+6. If you have a task like "Find the customer name and email with phone number X", you MUST use search_customer [Search customer with phone number X] to locate the customer, and then answer the query. Do NOT click on CUSTOMERS side panel.
+7. You MUST use Subroutine Actions whenever possible.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_customer_review = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_customer_review [query] asks you to navigate to the product page containing customer reviews.
+2. To navigate to a review, first click on REPORTS in the side panel
+3. Once you have clicked on REPORTS, and you see the Reports panel with Marketing, Sales, Reviews, Customers etc, click on By Products under Customers.
+4. Once you are in the Product Reviews Report, you need to locate the product by searching for it. Use the gridcell below Product to search for a product. Do not use other search boxes. Look at the example below where I show you how to search for Zoe in the correct gridcell.
+5. When searching for a product, search the first word only like Zoe, or Antonia or Chloe.
+6. Once the product shows up, click on 'Show Reviews'.
+7. Once all the reviews show up, return stop [N/A] to hand back control to the agent that queried you.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+ {
+"input": """
+OBJECTIVE:
+find_product_review [Show me the review of the customer who is the most unhappy with the style of Zoe products]
+OBSERVATION:
+Tab 0 (current): Product Reviews Report / Reviews / Reports / Magento Admin
+ table ''
+ row '| ID | Product | Reviews | Average | Average (Approved) | Last Review | Action |'
+ row '| --- | --- | --- | --- | --- | --- | --- |'
+ row "| textbox [1996] [required: False] | textbox [1998] [required: False] | textbox [2000] [required: False] | textbox [2002] [required: False] | textbox [2004] [required: False] | undefined undefined\ttextbox [2008] 'From' [required: False]\tbutton [3196] 'undefined'\ttextbox [2010] 'To' [required: False]\tbutton [3201] 'undefined' | |"
+ row 'https://localhost:7780/admin/review/product/index/productId/1620/'
+ row "| 1620 | Erica Evercool Sports Bra | 4 | 60.0000 | 60.0000 | Apr 19, 2023, 12:15:17 PM | Show Reviews\tlink [2792] 'Show
+ row 'https://localhost:7780/admin/review/product/index/productId/39/'
+ row "| 39 |
+URL:
+https://localhost:7780/admin/reports/report_review/product/
+PREVIOUS ACTIONS:
+click [1195]
+click [1531]
+""",
+"response": """
+REASON:
+Let's think step-by-step. We are in the Product Review Report page. We have to search for a Zoe product. To do so, I have to type in Zoe in a gridcell corresponding to the Product column. The Product column is the second column after ID. Therefore I have to find the second gridcell just below it. The first one is 1995, and the second one is 1997. In summary, the next action I will perform is type [1997] [Zoe] [1]
+ACTION:
+type [1997] [Zoe] [1]
+"""},
+]
+}
+
+find_order = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective find_order [query] asks you to navigate to the order page corresponding to the query
+2. To navigate to orders, first click on SALES in the side panel
+3. Once you have clicked on SALES, click on Orders.
+4. Once you are in the orders page, you have to use the 'Search by keyword' text box to search for your order. Always be sure to search first. For example, for find_order [Most recent pending order by Sarah Miller], search Sarah Miller.
+5. Click on View to open the right order.
+6. Once you are in the order page, as noted by "Order & Account Information", you MUST return stop [N/A] to hand back control to the agent that queried you. Do not go back to another page.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_customer = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. The objective search_customer [query] asks you to search for customer details corresponding to the query
+2. To navigate to customers, first click on CUSTOMERS in the side panel
+3. Once you have clicked on CUSTOMERS, click on All Customers.
+4. Once you are in the customers page, you have to use the 'Search by keyword' text box to search for your customer. Always be sure to search first. For example, for find_order [Search customer with phone number 8015551212], search 8015551212.
+5. If the page shows a number has already been searched, click on Clear All first. Then proceed with the search.
+6. Once you are done with the search, and the customer with matching query shows up, you MUST return stop [N/A] to hand back control to the agent that queried you. Do not go back to another page.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+shopping_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`search_order [question]`: Search orders to answer a question about my orders
+`find_products [query]`: Find products that match a query
+`search_reviews [query]`: Search reviews to answer a question about reviews
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+stop [Closed]
+hover [11]
+search_order [How much I spend on 4/19/2023 on shopping at One Stop Market?]
+list_products [List products from PS4 accessories category by ascending price]
+search_reviews [List out reviewers, if exist, who mention about ear cups being small]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. First check thhe OBJECTIVE. If the OBJECTIVE is a question about my orders, you MUST use search_order [question] to answer the question. For example,
+a. search_order [How much I spend on ...?]
+b. search_order [What is the size of the picture frame I bought Sep 2022?]
+c. search_order [Change the delivery address for my most recent order]
+Do not click on MyAccount directly!
+Do not try to solve the task without using search_order as it contains specific instructions on how to solve it.
+2. Once you call the search_order [] subroutine, the response is stored in PREVIOUS ACTIONS. For example,
+$0 = search_order [How much I spend on 4/19/2023 on shopping at One Stop Market?]
+means that the response was $0. In that case, return the answer directly, e.g. stop [$0]
+If the response was N/A, reply stop [N/A]
+3. If the OBJECTIVE is a question about listing / showing products, you MUST use list_products. For example,
+a. list_products [List products from PS4 accessories category by ascending price]
+4. If the OBJECTIVE requires you to retrieve details about a particular order you placed liked SKU, you MUST first use search_order [] to retrieve the SKU
+a. If the OBJECTIVE is "Fill the "contact us" form in the site for a refund on the bluetooth speaker I bought ... Also, ensure to include the order number #161 and the product SKU."
+you must first issue search_order [Give me the SKU of bluetooth speaker from order number #161]
+b. If the OBJECTIVE is "Draft a refund message via their "contact us" form for the phone screen protector I bought March 2023. It broke after three days of use. The shop requires the order id, the reason and the amount to refund in the message."
+you must first issue search_order [Give me the order id and the amount for the phone screen protector I bought March 2023.]
+5. If the OBJECTIVE is about reviews for the product, you MUST use search_reviews. For example,
+a. search_reviews [List out reviewers, if exist, who mention about ear cups being small]
+b. search_reviews [What are the main criticisms of this product? Please extract the relevant sentences]
+6. In your REASON, you MUST specify if any of the general instructions above apply that would affect the action you choose.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_order = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`note [content]`: Use this to make a personal note of some content you would like to remember. This shows up in your history of previous actions so you can refer to it.
+`go_back`: Navigate to the previously viewed page.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+note [Spent $10 on 4/1/2024]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. Navigate to My Account, then My Orders to access all orders
+2. The orders are sorted by descending date. Use Page Next to navigate to orders placed at an earlier date than displayed. Use Page Previous to navigate to orders at a later date than displayed.
+click [id] corresponding to Page Previous to go to a later date
+click [id] corresponding to Page Next to go to a earlier date
+3. If you don't see an order for a date, and the first order on the page is after the date, the last order on the page is before the date, then it means there is no order for the date. No point navigating to previous or next pages.
+4. If the question is how much did I spend on a date, and I didn't spend anything, return stop [$0]
+5. If the status of an order shows cancelled, that means I did not spend that money
+6. If you have to find the total amount you spent on orders that span multiple pages, use note [Spent $10 on 4/1/2024] to make a personal note before moving on to the next page. When you are done, you can look at PREVIOUS ACTIONS to find all notes.
+7. When you are adding numbers, work out each addition step by step in REASON.
+8. Use go_back to go back to a previous page from an order.
+But before you do, use note [] to make a note that you checked the page, e.g.
+note [Checked order on 11/29/2023, no picture frame.]
+9. If you are asked to change the delivery address on an order, you can't. Reply stop [N/A]
+10. If you are in an order page and need to go back, issue go_back.
+Don't click on My Orders else you have to start from all over again.
+11. Do not keep visiting the same order page over and over again!
+To prevent this, whenever you visit a page, always make a note. For example note [Nothing relevant purchased on September 29, 2022]
+See note [] to see what dates you have visit, and be sure to not visit that page again.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+list_products = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id. Use this whenever any element has the field hasPopup: menu
+`goto [url]`: Navigate to a specific URL. Use this when needing to sort by price. Refer to instructions below.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [N/A]
+hover [77]
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. To find a product category, you MUST use hover [id] to expand the popup Menu till you find the leaf element that has no popup. Then click [id] on the leaf element.
+For exmaple, to find PS 4 accessories you must hover over Video Games, then hover over Playstation 4, then click on Accessories.
+2. To sort current list of products by price and in ascending order, you MUST use the goto [url] action by appending ?product_list_order=price to the current URL. For example:
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+then issue goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price]
+3. To sort in descending order, you MUST use the goto [url] action by appending ?product_list_order=price&product_list_dir=desc, e.g.
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?product_list_order=price&product_list_dir=desc]
+4. To list all items less than a particular price, e.g. $25, you MUST use the goto [url] action by appending ?price=0-25
+If URL is https://localhost:7770/video-games/playstation-4/accessories.html
+goto [https://localhost:7770/video-games/playstation-4/accessories.html?price=0-25]
+5. Once you are done in stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_reviews = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`note [content]`: Use this to make a personal note of some content you would like to remember. This shows up in your history of previous actions so you can refer to it.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+note [Reviewer X made comment Y]
+stop [N/A]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. To find the list of reviews, search for a link with Reviewers.
+2. If you have to list multiple reviewers, use `note [Reviewer X made comment Y; Reviewer A made comment B; ...]` to make a personal note. When you are done, you can look at PREVIOUS ACTIONS to find all notes. In stop [], make sure you answer the question in the OBJECTIVE.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+maps_agent = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Subroutine Actions:
+`find_directions [query]`: Find directions between two locations to answer the query
+`search_nearest_place [query]`: Find places near a given location
+
+Example actions:
+click [7]
+type [15] [Carnegie Mellon University] [1]
+find_directions [Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University]
+search_nearest_place [Tell me the closest cafe(s) to CMU Hunt library]
+
+You will be provided with the following,
+ OBJECTIVE:
+ The goal you need to achieve.
+ OBSERVATION:
+ A simplified text description of the current browser content, without formatting elements.
+ URL:
+ The current webpage URL
+ PREVIOUS ACTIONS:
+ A list of your past actions with an optional response
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+ REASON:
+ Your reason for selecting the action below
+ ACTION:
+ Your action
+
+Please follow these general instructions:
+1. If the OBJECTIVE is about finding directions from A to B, you MUST use find_directions [] subroutine.
+e.g. find_directions [Check if the social security administration in pittsburgh can be reached in one hour by car from Carnegie Mellon University]
+2. If the OBJECTIVE is about searching nearest place to a location, you MUST use search_nearest_place [] subroutine.
+e.g. search_nearest_place [Tell me the closest restaurant(s) to Cohon University Center at Carnegie Mellon University]
+3. If the OBJECTIVE is to pull up a description, once that place appears in the sidepane, return stop [N/A]
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+find_directions = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [5h 47min]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. First click on "Find directions between two points", then enter From and To Fields, and click search.
+2. If you have to find directions to social security administration in Pittsburgh, search for it in a structured format like Social Security Administration, Pittsburgh.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
+
+search_nearest_place = {
+"instruction": """You are an AI assistant performing tasks on a web browser. To solve these tasks, you will issue specific actions.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Example actions:
+click [7]
+type [7] [Zoe] [1]
+stop [De Fer Coffee & Tea, La Prima Espresso, Rothberg's Roasters II, Cafe Phipps, La Prima Espresso, Starbucks]
+
+You will be provided with the following,
+OBJECTIVE:
+The goal you need to achieve.
+OBSERVATION:
+A simplified text description of the current browser content, without formatting elements.
+URL:
+The current webpage URL
+PREVIOUS ACTIONS:
+A list of your past actions
+
+You need to generate a response in the following format. Please issue only a single action at a time.
+REASON:
+Your reason for selecting the action below
+ACTION:
+Your action
+
+Please follow these instructions to solve the subtask:
+1. For searches that refer to CMU, e.g. "find cafes near CMU Hunt Library"
+a. You have to first center your map around a location. If you have to find cafes near CMU Hunt Library, the first step is to make sure the map is centered around Carnegie Mellon University. To do that, first search for Carnegie Mellon University and then click [] on a list of location that appears. You MUST click on the Carnegie Mellon University location to center the map. Else the map will not centered. E.g click [646]
+b. Now that your map is centered around Carnegie Mellon University, directly search for "cafes near Hunt Library". Do not include the word CMU in the search item.
+The word CMU cannot be parsed by maps and will result in an invalid search.
+c. When your search returns a list of elements, return them in a structured format like stop [A, B, C]
+2. For searches that don't refer to CMU
+a. No need to center the map. Directly search what is specified in OBJECTIVE, e.g. "bars near Carnegie Music Hall"
+b. When your search returns a list of elements, return them in a structured format like stop [A, B, C]
+3. Be sure to double check whether the OBJECTIVE has CMU or not and then choose between instruction 1 and 2.
+4. Remember that the word CMU cannot be typed in the search bar as it cannot be parsed by maps.
+5. Remember that if you want to center your map around Carnegie Mellon University, you have to click on it after you search for it. Check your PREVIOUS ACTIONS to confirm you have done so, e.g. click [646] should be in the previous actions.
+""",
+
+"input": """
+OBJECTIVE:
+{objective}
+OBSERVATION:
+{observation}
+URL:
+{url}
+PREVIOUS ACTIONS:
+{previous_actions}
+""",
+
+"response": "",
+
+"examples": [
+]
+}
diff --git a/webagents_step/utils/data_prep.py b/webagents_step/utils/data_prep.py
new file mode 100644
index 0000000..e3647eb
--- /dev/null
+++ b/webagents_step/utils/data_prep.py
@@ -0,0 +1,137 @@
+import os
+import json
+import pandas as pd
+import random
+import yaml
+import copy
+import numpy as np
+
+class yamlConfig:
+ def __init__(self, dictionary):
+ for key, value in dictionary.items():
+ if isinstance(value, dict):
+ value = yamlConfig(value)
+ setattr(self, key, value)
+
+class DotDict:
+ """access dictionary attributes with dot notation"""
+ def __init__(self, dictionary):
+ for key, value in dictionary.items():
+ if isinstance(value, dict):
+ value = DotDict(value)
+ setattr(self, key, value)
+
+ def to_dict(self):
+ regular_dict = {}
+ for key, value in self.__dict__.items():
+ if isinstance(value, DotDict):
+ regular_dict[key] = value.to_dict()
+ else:
+ regular_dict[key] = value
+ return regular_dict
+
+ def has_key(self, key):
+ """Check if the DotDict has a specific key"""
+ return hasattr(self, key)
+
+#####
+# Dataset processing functions
+#####
+
+def split_list_by_proportion(proportion, original_list):
+ list_size = len(original_list)
+ first_list_size = int(proportion * list_size)
+ shuffled_list = copy.copy(original_list)
+ random.shuffle(shuffled_list)
+ first_list = shuffled_list[:first_list_size]
+ second_list = shuffled_list[first_list_size:]
+ return first_list, second_list
+
+
+def examples_to_prompts(data_config, json_df, prompt_template, inference):
+ all_prompt_strings = []
+ for idx, row in json_df.iterrows():
+ try:
+ # read samples
+ json_filepath = os.path.join(data_config.basedir, str(row.logfile))
+ with open(json_filepath,'r') as json_file:
+ example_json = json.load(json_file)
+ all_prompt_strings.extend(convert_example_to_prompts(example_json, data_config, prompt_template, inference))
+ except FileNotFoundError:
+ print(f"File {data_config.basedir}/{row.logfile} not found. Skipping ...")
+ continue
+
+ return all_prompt_strings
+
+def convert_example_to_prompts(example_json, data_config, prompt_template, inference):
+ filled_prompt_strings = []
+ previous_actions = []
+ for step_ind, step in enumerate(example_json["trajectory"]):
+ # Fill in prompt components
+ prompt_components = copy.deepcopy(prompt_template)
+ prompt_components["input"] = prompt_components["input"].format(
+ objective = step["objective"],
+ observation = step["observation"],
+ url = step["url"],
+ previous_actions = "\n".join([str(step) for step in step["previous_actions"][-data_config.action_lookback:] if step is not None]) if "previous_actions" in step else "\n".join([str(action) for action in previous_actions[-data_config.action_lookback:] if action is not None])
+ )
+ previous_actions.append(step["action"])
+
+ prompt_components["response"] = convert_actions_and_reasons_to_response(step["reason"], step["action"])
+ filled_prompt_strings.append(create_prompt(prompt_components, inference=inference))
+ return filled_prompt_strings
+
+def convert_actions_and_reasons_to_response(reasons, actions):
+ response = f"""
+REASON:
+{reasons}
+ACTION:
+{actions}
+"""
+ return response
+
+def create_prompt(prompt_components, inference=False):
+ #If inference mode, do not fill in the response
+ if inference:
+ prompt_template = """
+[INST] <>
+{instruction}
+< >
+
+{input} [/INST]"""
+ return prompt_template.format(instruction = prompt_components["instruction"],
+ input = prompt_components["input"])
+ else:
+ prompt_template = """
+[INST] <>
+{instruction}
+< >
+
+{input} [/INST] {response}
+"""
+ return prompt_template.format(instruction = prompt_components["instruction"],
+ input = prompt_components["input"],
+ response = prompt_components["response"])
+
+def log_run(log_file, log_data, summary_file=None, summary_data=None, json_indent=4, verbose=1):
+ """
+ Logs demo data to a JSON file and optionally updates a summary CSV file.
+ """
+ # Write log data to JSON file
+ with open(log_file, 'w') as json_file:
+ json.dump(log_data, json_file, indent=json_indent)
+ if verbose:
+ print(f"Saved log to {log_file}")
+
+ # If summary data and file path are provided, update the summary
+ if summary_data and summary_file:
+ if os.path.exists(summary_file):
+ df_summary = pd.read_csv(summary_file)
+ else:
+ df_summary = pd.DataFrame()
+ df_summary = pd.concat([df_summary, pd.DataFrame([summary_data])], ignore_index=True)
+ df_summary.to_csv(summary_file, index=False)
+ if verbose:
+ print(f"Updated summary: {df_summary}")
+
+ return df_summary
\ No newline at end of file
diff --git a/webagents_step/utils/llm.py b/webagents_step/utils/llm.py
new file mode 100644
index 0000000..1aac13a
--- /dev/null
+++ b/webagents_step/utils/llm.py
@@ -0,0 +1,422 @@
+import openai
+import re
+import copy
+import torch
+from transformers import (
+ AutoModelForCausalLM,
+ AutoTokenizer,
+ BitsAndBytesConfig,
+ AutoModelForSeq2SeqLM,
+)
+import ctranslate2
+from time import sleep
+import tiktoken
+from openai import OpenAI
+
+import warnings
+warnings.simplefilter("ignore")
+
+input_token_cost_usd_by_model = {
+ "gpt-4-1106-preview": 0.01 / 1000,
+ "gpt-4": 0.03 / 1000,
+ "gpt-4-32k": 0.06 / 1000,
+ "gpt-3.5-turbo": 0.001 / 1000,
+ "gpt-3.5-turbo-instruct": 0.0015 / 1000,
+ "gpt-3.5-turbo-16k": 0.003 / 1000,
+ "babbage-002": 0.0016 / 1000,
+ "davinci-002": 0.012 / 1000,
+ "ada-v2": 0.0001 / 1000,
+}
+
+output_token_cost_usd_by_model = {
+ "gpt-4-1106-preview": 0.03 / 1000,
+ "gpt-4": 0.06 / 1000,
+ "gpt-4-32k": 0.12 / 1000,
+ "gpt-3.5-turbo": 0.002 / 1000,
+ "gpt-3.5-turbo-instruct": 0.002 / 1000,
+ "gpt-3.5-turbo-16k": 0.004 / 1000,
+ "babbage-002": 0.0016 / 1000,
+ "davinci-002": 0.012 / 1000,
+ "ada-v2": 0.0001 / 1000,
+}
+
+def fill_prompt_template(prompt_template, objective, observation, url, previous_history):
+ prompt = copy.deepcopy(prompt_template)
+ prompt["input"] = prompt["input"].replace("{objective}", objective)
+ if isinstance(observation, dict):
+ prompt["input"] = prompt["input"].replace("{observation}", observation["text"])
+ else:
+ prompt["input"] = prompt["input"].replace("{observation}", observation)
+ prompt["input"] = prompt["input"].replace("{url}", url)
+ prompt["input"] = prompt["input"].replace("{previous_actions}", previous_history)
+ return prompt
+
+def filter_quotes_if_matches_template(action):
+ if action is None:
+ return None
+
+ # Regex pattern to match the entire 'type [X] ["Y"]' template, allowing for Y to be digits as well
+ pattern = r'^type \[\d+\] \["([^"\[\]]+)"\]$'
+ # Check if the action matches the specific template
+ match = re.match(pattern, action)
+ if match:
+ # Extract the matched part that needs to be unquoted
+ y_part = match.group(1)
+ # Reconstruct the action string without quotes around Y
+ filtered_action = f'type [{match.group(0).split("[")[1].split("]")[0]}] [{y_part}]'
+ return filtered_action.strip() # filtered_action.split("\n")[0].strip()
+ else:
+ # Return the original action if it doesn't match the template
+ return action.strip() # action.split("\n")[0].strip()
+
+def parse_action_reason(model_response):
+ reason_match = re.search(r'REASON:\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL)
+ reason = reason_match.group(1) if reason_match else None
+
+ # action_match = re.search(r'ACTION:\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL)
+ action_match = re.search(r'(?:ACTION|ACTIONS):\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL)
+ action = action_match.group(1) if action_match else None
+
+ action = filter_quotes_if_matches_template(action)
+
+ return action, reason
+
+def construct_llm_message_hf(prompt, prompt_mode, model_type="llama2"):
+ if model_type == "llama2":
+ instruction = "[INST] " + prompt["instruction"]
+ else:
+ instruction = prompt["instruction"]
+
+ messages = [{"role": "system", "content": instruction}]
+
+ if prompt["examples"]:
+ messages.append({"role": "system", "content": "Here are a few examples:"})
+ for example in prompt["examples"]:
+ messages.append({"role": "system", "content": f"\n### Input:\n{example['input']}\n\n### Response:\n{example['response']}"})
+
+ if model_type == "llama2":
+ query = f"\nHere is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n[/INST]\n"
+ else:
+ query = f"\nHere is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"
+
+ messages.append({"role": "user", "content": query})
+ if prompt_mode == "chat":
+ return messages
+ elif prompt_mode == "completion":
+ all_content = ''.join(message['content'] for message in messages)
+ messages_completion = [{"role": "user", "content": all_content}]
+ return messages_completion
+
+def construct_llm_message_anthropic(prompt, plan_list=None, action_list=None):
+ if plan_list and action_list:
+ import os
+ from global_utils import CURRENT_DIR
+
+ assert len(plan_list) > 0 and len(action_list) > 0
+ plan_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "plan_instructions", f"{p}.txt"), "r").readlines()) for p in plan_list])
+ action_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "action_instructions", f"{a}.txt"), "r").readlines()) for a in action_list])
+ prompt["instruction"] = prompt["instruction"].replace("{plan_instructions}", plan_instructions)
+ prompt["instruction"] = prompt["instruction"].replace("{action_instructions}", action_instructions)
+
+ system_message = prompt["instruction"]
+
+ if prompt["examples"]:
+ system_message += f"\n\n## Here are a few examples:"
+ for i, example in enumerate(prompt["examples"]):
+ example_input = example["input"]
+ example_response = example["response"]
+ if "example_format" in prompt.keys():
+ system_message += "\n\n"
+ system_message += prompt["example_format"].replace("{i}", i).replace("{example_input}", example_input).replace("{example_response}", example_response)
+ else:
+ system_message += f"\n\n| Example {i}\n\n### Input:\n{example_input}\n\n### Response: Let's think step by step.\n{example_response}"
+
+ if "input_format" in prompt.keys():
+ if "{visual_observation}" in prompt.keys():
+ from claude import arrange_message_for_claude
+ text = prompt["input_format"].replace("{input}", prompt['input'])
+ text_prior, text_subsequent = text.split("{visual_observation}")
+ messages = arrange_message_for_claude([("text", text_prior), ("image", prompt["{visual_observation}"]), ("text", text_subsequent)])
+ else:
+ messages = [{"role": "user", "content": [{"type": "text", "text": prompt["input_format"].replace("{input}", prompt['input'])}]}]
+ else:
+ if "{visual_observation}" in prompt.keys():
+ pass
+ else:
+ messages = [{"role": "user", "content": [{"type": "text", "text": f"## Here is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"}]}]
+
+ return system_message, messages
+
+def construct_llm_message_openai(prompt, prompt_mode, plan_list=None, action_list=None):
+ if not (plan_list and action_list):
+ messages = [{"role": "system", "content": prompt["instruction"]}]
+
+ if prompt["examples"]:
+ messages.append({"role": "system", "content": "Here are a few examples:"})
+ for example in prompt["examples"]:
+ messages.append({"role": "system", "content": f"\n### Input:\n{example['input']}\n\n### Response:\n{example['response']}"})
+
+ messages.append({"role": "user", "content": f"Here is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"})
+ if prompt_mode == "chat":
+ return messages
+ elif prompt_mode == "completion":
+ all_content = ''.join(message['content'] for message in messages)
+ messages_completion = [{"role": "user", "content": all_content}]
+ return messages_completion
+ import os
+ from global_utils import CURRENT_DIR
+
+ assert len(plan_list) > 0 and len(action_list) > 0
+ plan_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "plan_instructions", f"{p}.txt"), "r").readlines()) for p in plan_list])
+ action_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "action_instructions", f"{a}.txt"), "r").readlines()) for a in action_list])
+ prompt["instruction"] = prompt["instruction"].replace("{plan_instructions}", plan_instructions)
+ prompt["instruction"] = prompt["instruction"].replace("{action_instructions}", action_instructions)
+
+ messages = [{"role": "system", "content": prompt["instruction"]}]
+
+ if prompt["examples"]:
+ messages.append({"role": "system", "content": "## Here are a few examples:"})
+ for i, example in enumerate(prompt["examples"]):
+ example_input = example["input"]
+ example_response = example["response"]
+ messages.append({"role": "system", "content": f"| Example {i}\n\n### Input:\n{example_input}\n\n### Response: Let's think step by step.\n{example_response}"})
+
+ if "input_format" in prompt.keys():
+ messages.append({"role": "user", "content": prompt["input_format"].replace("{input}", prompt['input'])})
+ else:
+ messages.append({"role": "user", "content": f"## Here is the current Input. Please respond with PLAN, REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"})
+ if prompt_mode == "chat":
+ return messages
+ elif prompt_mode == "completion":
+ all_content = ''.join(message['content'] for message in messages)
+ messages_completion = [{"role": "user", "content": all_content}]
+ return messages_completion
+
+def call_anthropic_llm(system_message, messages, model="anthropic.claude-3-haiku-20240307-v1:0", **model_kwargs):
+ # Use the native inference API to send a text message to Anthropic Claude.
+
+ import boto3
+ import json
+
+ # Create a Bedrock Runtime client in the AWS Region of your choice.
+ client = boto3.client("bedrock-runtime", region_name="us-east-1")
+ print(system_message, file=open("trash.txt", "a"))
+ print("\n".join(item["content"][0]["text"] for item in messages), end="\n"+"#"*100+"\n", file=open("trash.txt", "a"))
+ native_request = {
+ "anthropic_version": "bedrock-2023-05-31",
+ "max_tokens": 512,
+ "temperature": 0.5,
+ "system": system_message,
+ "messages": messages,
+ }
+
+ # Convert the native request to JSON.
+ request = json.dumps(native_request)
+
+ try:
+ # Invoke the model with the request.
+ response = client.invoke_model(modelId=model, body=request)
+
+ except Exception as e:
+ raise KeyError(f"ERROR: Can't invoke '{model}'. Reason: {e}")
+
+ # Decode the response body.
+ model_response = json.loads(response["body"].read())
+
+ # Extract and print the response text.
+ response_text = model_response["content"][0]["text"]
+ return response_text
+
+def call_openai_llm(messages, model="gpt-3.5-turbo", **model_kwargs):
+ """
+ Sends a request with a chat conversation to OpenAI's chat API and returns a response.
+
+ Parameters:
+ messages (list)
+ A list of dictionaries containing the messages to send to the chatbot.
+ model (str)
+ The model to use for the chatbot. Default is "gpt-3.5-turbo".
+ temperature (float)
+ The temperature to use for the chatbot. Defaults to 0. Note that a temperature
+ of 0 does not guarantee the same response (https://platform.openai.com/docs/models/gpt-3-5).
+
+ Returns:
+ response (Optional[dict])
+ The response from OpenAI's chat API, if any.
+ """
+ # client = OpenAI()
+ temperature = model_kwargs.get('temperature', 0.7)
+ top_p = model_kwargs.get('top_p', 1.0)
+ n = model_kwargs.get('n', 1)
+
+ for m in messages:
+ print(m["content"], file=open("trash.txt", "a"))
+ print("*"*100, file=open("trash.txt", "a"))
+
+ num_attempts = 0
+ while True:
+ if num_attempts >= 10:
+ raise ValueError("OpenAI request failed.")
+ try:
+ if model=="text-davinci-003":
+ response = openai.Completion.create(
+ model=model,
+ prompt=messages[0]["content"],
+ temperature=temperature,
+ top_p=top_p,
+ n=n,
+ max_tokens=128)
+ return response.choices[0].text.strip()
+
+ response = OpenAI().chat.completions.create(
+ model=model,
+ messages=messages,
+ temperature=temperature,
+ top_p=top_p,
+ n=n
+ )
+ return response.choices[0].message.content.strip()
+ except openai.AuthenticationError as e:
+ print(e)
+ return None
+ except openai.RateLimitError as e:
+ print(e)
+ print("Sleeping for 10 seconds...")
+ sleep(10)
+ num_attempts += 1
+ except Exception as e:
+ print(e)
+ print("Sleeping for 10 seconds...")
+ sleep(10)
+ num_attempts += 1
+
+def get_num_tokens(text: str, model_name: str) -> int:
+ tokenizer = tiktoken.encoding_for_model(model_name=model_name)
+ return len(tokenizer.encode_ordinary(text))
+
+def calculate_cost_openai(messages: str, response: str, model_name: str) -> int:
+ input_text = " ".join([msg["content"] for msg in messages])
+ num_input_tokens = get_num_tokens(input_text, model_name)
+ num_output_tokens = get_num_tokens(response, model_name)
+
+ input_token_cost = input_token_cost_usd_by_model.get(model_name, None)
+ output_token_cost = output_token_cost_usd_by_model.get(model_name, None)
+ if input_token_cost is None or output_token_cost is None:
+ print(f"[calculate_cost_openai] unknown model {model_name}")
+ return 0
+ return num_input_tokens * input_token_cost + num_output_tokens * output_token_cost
+
+def load_tokenizer(mpath, context_size):
+ tokenizer = AutoTokenizer.from_pretrained(mpath, return_token_type_ids=False)
+ # tokenizer.pad_token = tokenizer.eos_token
+ # tokenizer.pad_token_id = tokenizer.eos_token_id
+ # tokenizer.model_max_length = context_size
+ # tokenizer.padding_side = "right"
+ # tokenizer.truncation_side = "left"
+ # tokenizer.add_eos_token = True
+ return tokenizer
+
+def load_model(mpath, dtype, device="cuda", context_len=4096, is_seq2seq=False, ct2_mpath=None):
+ if is_seq2seq:
+ model_loader = AutoModelForSeq2SeqLM
+ else:
+ model_loader = AutoModelForCausalLM
+
+ if dtype == "bf16":
+ model = model_loader.from_pretrained(
+ mpath,
+ max_position_embeddings=context_len,
+ low_cpu_mem_usage=True,
+ torch_dtype=torch.bfloat16,
+ device_map="balanced_low_0",
+ )
+ elif dtype == "4bit":
+ model = model_loader.from_pretrained(
+ mpath,
+ max_position_embeddings=context_len,
+ low_cpu_mem_usage=True,
+ load_in_4bit=True,
+ device_map="auto",
+ )
+ elif dtype == "4bit-optimized":
+ bnb_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_compute_dtype=torch.bfloat16,
+ )
+ model = model_loader.from_pretrained(
+ mpath,
+ use_cache=True,
+ device_map="auto",
+ quantization_config=bnb_config,
+ max_position_embeddings=context_len,
+ )
+ elif dtype == "8bit":
+ model = model_loader.from_pretrained(
+ mpath,
+ max_position_embeddings=context_len,
+ low_cpu_mem_usage=True,
+ load_in_8bit=True,
+ device_map="auto",
+ )
+ elif dtype == "ct2":
+ assert ct2_mpath is not None
+ model = ctranslate2.Generator(ct2_mpath, device=device)
+
+ return model
+
+# @torch.no_grad()
+# def generate_prediction(
+# inputs,
+# model,
+# tokenizer,
+# max_new_tokens,
+# is_seq2seq=False,
+# **kwargs,
+# # num_beams,
+# # do_sample,
+# # no_repeat_ngram_size,
+# # temperature,
+# # top_k,
+# # top_p,
+# ):
+# input_ids = tokenizer(inputs, return_tensors="pt", truncation=True, max_length=tokenizer.model_max_length - max_new_tokens).input_ids
+
+# outputs = model.generate(
+# input_ids=input_ids.cuda(),
+# max_new_tokens=max_new_tokens,
+# **kwargs,
+# ).cpu()
+
+# torch.cuda.empty_cache()
+# if not is_seq2seq:
+# outputs = outputs[:, input_ids.shape[1] :]
+
+# prediction = [
+# p.split(tokenizer.pad_token, 1)[0]
+# for p in tokenizer.batch_decode(outputs, skip_special_tokens=True)
+# ][0].strip()
+
+# return prediction
+
+def generate_prediction(
+ inputs,
+ model,
+ tokenizer,
+ **kwargs,
+):
+ inputs = tokenizer([inputs], return_tensors='pt', truncation=True, add_special_tokens=False).to(model.device)
+
+ # if torch.cuda.is_available():
+ # inputs = inputs.to('cuda')
+ outputs = model.generate(
+ input_ids=inputs['input_ids'],
+ attention_mask=inputs['attention_mask'],
+ **kwargs,
+ )
+
+ outputs = outputs[:, inputs.input_ids.shape[1] :]
+ prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+ return prediction
\ No newline at end of file
diff --git a/webagents_step/utils/stack.py b/webagents_step/utils/stack.py
new file mode 100644
index 0000000..8944891
--- /dev/null
+++ b/webagents_step/utils/stack.py
@@ -0,0 +1,22 @@
+class Stack:
+ def __init__(self):
+ self.items = []
+
+ def is_empty(self):
+ return len(self.items) == 0
+
+ def push(self, item):
+ self.items.append(item)
+
+ def pop(self):
+ if self.is_empty():
+ raise IndexError("pop from empty stack")
+ return self.items.pop()
+
+ def peek(self):
+ if self.is_empty():
+ raise IndexError("peek from empty stack")
+ return self.items[-1]
+
+ def size(self):
+ return len(self.items)
\ No newline at end of file
diff --git a/webarena/agent/__init__.py b/webarena/agent/__init__.py
new file mode 100644
index 0000000..9028d30
--- /dev/null
+++ b/webarena/agent/__init__.py
@@ -0,0 +1,8 @@
+from .agent import (
+ Agent,
+ PromptAgent,
+ TeacherForcingAgent,
+ construct_agent,
+)
+
+__all__ = ["Agent", "TeacherForcingAgent", "PromptAgent", "construct_agent"]
diff --git a/webarena/agent/agent.py b/webarena/agent/agent.py
new file mode 100644
index 0000000..490d08e
--- /dev/null
+++ b/webarena/agent/agent.py
@@ -0,0 +1,227 @@
+import argparse
+import json
+from typing import Any
+
+import tiktoken
+from beartype import beartype
+
+from agent.prompts import *
+from browser_env import Trajectory
+from browser_env.actions import (
+ Action,
+ ActionParsingError,
+ create_id_based_action,
+ create_none_action,
+ create_playwright_action,
+)
+from browser_env.utils import Observation, StateInfo
+from llms import (
+ call_llm,
+ generate_from_huggingface_completion,
+ generate_from_openai_chat_completion,
+ generate_from_openai_completion,
+ lm_config,
+)
+from llms.tokenizers import Tokenizer
+
+
+class Agent:
+ """Base class for the agent"""
+
+ def __init__(self, *args: Any) -> None:
+ pass
+
+ def next_action(
+ self, trajectory: Trajectory, intent: str, meta_data: Any
+ ) -> Action:
+ """Predict the next action given the observation"""
+ raise NotImplementedError
+
+ def check_action(
+ self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any], target_action: str
+ ) -> Action:
+ """Predict the next action given the observation"""
+ raise NotImplementedError
+
+ def reset(
+ self,
+ test_config_file: str,
+ ) -> None:
+ raise NotImplementedError
+
+
+class TeacherForcingAgent(Agent):
+ """Agent that follows a pre-defined action sequence"""
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def set_action_set_tag(self, tag: str) -> None:
+ self.action_set_tag = tag
+
+ def set_actions(self, action_seq: str | list[str]) -> None:
+ if isinstance(action_seq, str):
+ action_strs = action_seq.strip().split("\n")
+ else:
+ action_strs = action_seq
+ action_strs = [a.strip() for a in action_strs]
+
+ actions = []
+ for a_str in action_strs:
+ try:
+ if self.action_set_tag == "playwright":
+ cur_action = create_playwright_action(a_str)
+ elif self.action_set_tag == "id_accessibility_tree":
+ cur_action = create_id_based_action(a_str)
+ else:
+ raise ValueError(
+ f"Unknown action type {self.action_set_tag}"
+ )
+ except ActionParsingError as e:
+ cur_action = create_none_action()
+
+ cur_action["raw_prediction"] = a_str
+ actions.append(cur_action)
+
+ self.actions: list[Action] = actions
+
+ def next_action(
+ self, trajectory: Trajectory, intent: str, meta_data: Any
+ ) -> Action:
+ """Predict the next action given the observation"""
+ return self.actions.pop(0)
+
+ def reset(
+ self,
+ test_config_file: str,
+ ) -> None:
+ with open(test_config_file) as f:
+ ref_actions = json.load(f)["reference_action_sequence"]
+ tag = ref_actions["action_set_tag"]
+ action_seq = ref_actions["action_sequence"]
+ self.set_action_set_tag(tag)
+ self.set_actions(action_seq)
+
+
+class PromptAgent(Agent):
+ """prompt-based agent that emits action given the history"""
+
+ @beartype
+ def __init__(
+ self,
+ action_set_tag: str,
+ lm_config: lm_config.LMConfig,
+ prompt_constructor: PromptConstructor,
+ ) -> None:
+ super().__init__()
+ self.lm_config = lm_config
+ self.prompt_constructor = prompt_constructor
+ self.action_set_tag = action_set_tag
+
+ def set_action_set_tag(self, tag: str) -> None:
+ self.action_set_tag = tag
+
+ @beartype
+ def next_action(
+ self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any]
+ ) -> Action:
+ prompt = self.prompt_constructor.construct(
+ trajectory, intent, meta_data
+ )
+ lm_config = self.lm_config
+ n = 0
+ while True:
+ response = call_llm(lm_config, prompt)
+ force_prefix = self.prompt_constructor.instruction[
+ "meta_data"
+ ].get("force_prefix", "")
+ response = f"{force_prefix}{response}"
+ n += 1
+ try:
+ parsed_response = self.prompt_constructor.extract_action(
+ response
+ )
+ if self.action_set_tag in ["id_html_tree", "id_html_nasc_tree", "id_accessibility_tree"]:
+ action = create_id_based_action(parsed_response)
+ elif self.action_set_tag == "playwright":
+ action = create_playwright_action(parsed_response)
+ else:
+ raise ValueError(
+ f"Unknown action type {self.action_set_tag}"
+ )
+ action["raw_prediction"] = response
+ break
+ except ActionParsingError as e:
+ if n >= lm_config.gen_config["max_retry"]:
+ action = create_none_action()
+ action["raw_prediction"] = response
+ break
+
+ return action
+
+ def check_action(
+ self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any], target_action: str
+ ) -> Action:
+ prompt = self.prompt_constructor.construct(
+ trajectory, intent, meta_data
+ )
+ lm_config = self.lm_config
+ n = 0
+
+ # agent will retry if the action is not parsed correctly
+ while True:
+ response = target_action
+ force_prefix = self.prompt_constructor.instruction[
+ "meta_data"
+ ].get("force_prefix", "")
+ response = f"{force_prefix}{response}"
+ n += 1
+ try:
+ parsed_response = self.prompt_constructor.extract_action(
+ response
+ )
+ if self.action_set_tag in ["id_accessibility_tree", "id_html_tree", "id_html_nasc_tree"]:
+ action = create_id_based_action(parsed_response)
+ elif self.action_set_tag == "playwright":
+ action = create_playwright_action(parsed_response)
+ else:
+ raise ValueError(
+ f"Unknown action type {self.action_set_tag}"
+ )
+ action["raw_prediction"] = response
+ break
+ except ActionParsingError as e:
+ if n >= lm_config.gen_config["max_retry"]:
+ action = create_none_action()
+ action["raw_prediction"] = response
+ break
+
+ return prompt, action
+
+ def reset(self, test_config_file: str) -> None:
+ pass
+
+
+def construct_agent(args: argparse.Namespace) -> Agent:
+ llm_config = lm_config.construct_llm_config(args)
+
+ agent: Agent
+ if args.agent_type == "teacher_forcing":
+ agent = TeacherForcingAgent()
+ elif args.agent_type == "prompt":
+ with open(args.instruction_path) as f:
+ constructor_type = json.load(f)["meta_data"]["prompt_constructor"]
+ tokenizer = Tokenizer(args.provider, args.model)
+ prompt_constructor = eval(constructor_type)(
+ args.instruction_path, lm_config=llm_config, tokenizer=tokenizer
+ )
+ agent = PromptAgent(
+ action_set_tag=args.action_set_tag,
+ lm_config=llm_config,
+ prompt_constructor=prompt_constructor,
+ )
+ else:
+ raise NotImplementedError(
+ f"agent type {args.agent_type} not implemented"
+ )
+ return agent
diff --git a/webarena/agent/prompts/__init__.py b/webarena/agent/prompts/__init__.py
new file mode 100644
index 0000000..3f3caba
--- /dev/null
+++ b/webarena/agent/prompts/__init__.py
@@ -0,0 +1 @@
+from .prompt_constructor import *
diff --git a/webarena/agent/prompts/jsons/new_action_prompt.json b/webarena/agent/prompts/jsons/new_action_prompt.json
new file mode 100644
index 0000000..d477d0c
--- /dev/null
+++ b/webarena/agent/prompts/jsons/new_action_prompt.json
@@ -0,0 +1,21 @@
+{
+ "intro": "",
+ "examples": [],
+ "template": " {html} \n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description.\nYour goal is to complete the task. You can use the provided functions below to interact with the current webpage.\n\n#Provided functions:\ndef click(element_id: str) -> None:\n \"\"\"\n Click on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef hover(element_id: str) -> None:\n \"\"\"\n Hover on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef select(element_id: str, option: str) -> None:\n \"\"\"\n Select an option from a dropdown.\n\n Args:\n element_id: The id of the element.\n option: Value of the option to select.\n \"\"\"\n\ndef type_string(element_id: str, content: str, press_enter: bool) -> None:\n \"\"\"\n Type a string into the element with the specified id.\n\n Args:\n element_id: The id of the element.\n content: The string to type.\n press_enter: Whether to press enter after typing the string.\n \"\"\"\n\ndef scroll_page(direction: Literal['up', 'down']) -> None:\n \"\"\"\n Scroll down/up one page.\n\n Args:\n direction: The direction to scroll.\n \"\"\"\n\ndef go(direction: Literal['forward', 'backward']) -> None:\n \"\"\"\n Go forward/backward\n\n Args:\n direction: The direction to go to.\n \"\"\"\n\ndef jump_to(url: str, new_tab: bool) -> None:\n \"\"\"\n Jump to the specified url.\n\n Args:\n url: The url to jump to.\n new_tab: Whether to open the url in a new tab.\n \"\"\"\n\ndef switch_tab(tab_index: int) -> None:\n \"\"\"\n Switch to the specified tab.\n\n Args:\n tab_index: The index of the tab to switch to.\n \"\"\"\n\ndef user_input(message: str) -> str:\n \"\"\"\n Wait for user input.\n\n Args:\n message: The message to display to the user.\n\n Returns: The user input.\n \"\"\"\n\ndef finish(answer: Optional[str]) -> None:\n \"\"\"\n Finish the task (optionally with an answer).\n\n Args:\n answer: The answer to the task.\n \"\"\"\n\n#Previous commands: {previous_action}\n\n#Window tabs: {tabs}\n\n#Current viewport (pages): {position}\n\n#Task: {objective}\n\nYou should output one command to interact to the currrent webpage.\nYou should add a brief comment to your command to explain your reasoning and thinking process.\n",
+ "finale": "",
+ "meta_data": {
+ "observation": "html",
+ "action_type": "id_html_nasc_tree",
+ "keywords": [
+ "url",
+ "html",
+ "objective",
+ "position",
+ "previous_action",
+ "tabs"
+ ],
+ "prompt_constructor": "NewASPromptConstructor",
+ "answer_phrase": "",
+ "action_splitter": "#"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/p_cot_id_actree_2s.json b/webarena/agent/prompts/jsons/p_cot_id_actree_2s.json
new file mode 100644
index 0000000..9d2eae4
--- /dev/null
+++ b/webarena/agent/prompts/jsons/p_cot_id_actree_2s.json
@@ -0,0 +1,27 @@
+{
+ "intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
+ "examples": [
+ [
+ "OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
+ "Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
+ ],
+ [
+ "OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
+ "Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
+ ]
+ ],
+ "template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": [
+ "url",
+ "objective",
+ "observation",
+ "previous_action"
+ ],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json b/webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json
new file mode 100644
index 0000000..6b0f23f
--- /dev/null
+++ b/webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json
@@ -0,0 +1,27 @@
+{
+ "intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
+ "examples": [
+ [
+ "OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
+ "Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
+ ],
+ [
+ "OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
+ "Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
+ ]
+ ],
+ "template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": [
+ "url",
+ "objective",
+ "observation",
+ "previous_action"
+ ],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/p_direct_id_actree_2s.json b/webarena/agent/prompts/jsons/p_direct_id_actree_2s.json
new file mode 100644
index 0000000..d336a03
--- /dev/null
+++ b/webarena/agent/prompts/jsons/p_direct_id_actree_2s.json
@@ -0,0 +1,26 @@
+{
+ "intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
+ "examples": [
+ [
+ "OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
+ "```stop [$279.49]```"
+ ],
+ [
+ "OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
+ "```type [164] [restaurants near CMU] [1]```"
+ ]
+ ],
+ "template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": [
+ "url",
+ "objective",
+ "observation",
+ "previous_action"
+ ],
+ "prompt_constructor": "DirectPromptConstructor",
+ "action_splitter": "```"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/p_direct_id_actree_2s_no_na.json b/webarena/agent/prompts/jsons/p_direct_id_actree_2s_no_na.json
new file mode 100644
index 0000000..ac3306f
--- /dev/null
+++ b/webarena/agent/prompts/jsons/p_direct_id_actree_2s_no_na.json
@@ -0,0 +1,27 @@
+{
+ "intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective.",
+ "examples": [
+ [
+ "OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
+ "```stop [$279.49]```"
+ ],
+ [
+ "OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
+ "```type [164] [restaurants near CMU] [1]```"
+ ]
+ ],
+ "template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": [
+ "url",
+ "objective",
+ "observation",
+ "previous_action"
+ ],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/p_direct_id_actree_3s_llama.json b/webarena/agent/prompts/jsons/p_direct_id_actree_3s_llama.json
new file mode 100644
index 0000000..f87f09f
--- /dev/null
+++ b/webarena/agent/prompts/jsons/p_direct_id_actree_3s_llama.json
@@ -0,0 +1,32 @@
+{
+ "intro": "You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\n\nYou can only issue one action at a time",
+ "examples": [
+ [
+ "Observation:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t[1749] StaticText '$279.49'\n\t[1757] button 'Add to Cart'\n\t[1760] button 'Add to Wish List'\n\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nObjective: What is the price of HP Inkjet Fax Machine\nPrevious action: None",
+ "```stop [$279.49]```"
+ ],
+ [
+ "Observation:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nObjective: Show me the restaurants near CMU\nPrevious action: None",
+ "```type [164] [restaurants near CMU] [1]```"
+ ],
+ [
+ "Observation:\n[2036] button 'Sort by: New' hasPopup: menu expanded: False\n\t[587] link 'US Marine\u2019s adoption of Afghan war orphan voided'\n\t\t[989] time 'March 30, 2023 at 15:03:48 AM UTC'\n\t[602] link 'York student uses AI chatbot to get parking fine revoked'\n\t\t[1025] time 'March 15, 2023 at 7:48:34 AM UTC'\n\t[617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'\n\t\t[1025] time 'March 2, 2023 at 3:46:01 AM UTC'\nURL: http://reddit.com/f/news/new\nObjective: Open the most recent post that was published prior to March 1st.\nPrevious action: None",
+ "```scroll [down]```"
+ ]
+ ],
+ "template": "Observation:\n{observation}\nURL: {url}\nObjective: {objective}\nPrevious action: {previous_action}",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": [
+ "url",
+ "objective",
+ "observation",
+ "previous_action"
+ ],
+ "prompt_constructor": "DirectPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```",
+ "force_prefix": "```"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/jsons/test_prompt.json b/webarena/agent/prompts/jsons/test_prompt.json
new file mode 100644
index 0000000..e227938
--- /dev/null
+++ b/webarena/agent/prompts/jsons/test_prompt.json
@@ -0,0 +1,20 @@
+{
+ "intro": "",
+ "examples": [],
+ "template": " {html} \n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description. \nYour goal is to complete the task. You can perform the specified operations below to interact with the webpage.\n\n#Valid operations: - #Click# id: Click on the element with the specified id\n- #Scroll_up#: Scroll up 1 page.\n- #Scroll_down#: Scroll down 1 page.\n- #Go_backward#: Go back to the previous page.\n- #Go_forward#: Go forward to the next page.\n- #Hover# id: Hover over the element with the specified id.\n- #Type# id \"text\": Type in the text at the element with the specified id.\n- #Select# id \"option\": Select the option at the element with the specified id.\n- #Record# \"content\": Mark content that is useful in answering the question.\n- #Answer# \"text\": output the text as the answer to the user.\n- #Exit#: Complete the task and exit the program.\n\n#Current viewport position: {position}\n\n#Previous Operation: {previous_action}\n\n#Task: {objective}\n",
+ "finale": "\nYour output SHOULD be in the following format:\n#Operation: {Next operation to perform}\n",
+ "meta_data": {
+ "observation": "html",
+ "action_type": "id_html_tree",
+ "keywords": [
+ "url",
+ "html",
+ "objective",
+ "position",
+ "previous_action"
+ ],
+ "prompt_constructor": "MyPromptConstructor",
+ "answer_phrase": "",
+ "action_splitter": "#"
+ }
+}
\ No newline at end of file
diff --git a/webarena/agent/prompts/prompt_constructor.py b/webarena/agent/prompts/prompt_constructor.py
new file mode 100644
index 0000000..039b87c
--- /dev/null
+++ b/webarena/agent/prompts/prompt_constructor.py
@@ -0,0 +1,683 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, TypedDict
+
+from browser_env import Action, ActionParsingError, Trajectory
+from browser_env.env_config import URL_MAPPINGS
+from browser_env.utils import StateInfo
+from llms import lm_config
+from llms.tokenizers import Tokenizer
+from llms.utils import APIInput
+
+
+class Instruction(TypedDict):
+ """Instruction for constructing prompt"""
+
+ intro: str
+ examples: list[tuple[str, str]]
+ template: str
+ meta_data: dict[str, Any]
+
+
+class PromptConstructor(object):
+ def __init__(
+ self,
+ instruction_path: str | Path,
+ lm_config: lm_config.LMConfig,
+ tokenizer: Tokenizer,
+ ):
+ self.instruction_path = Path(instruction_path)
+ self.obs_modality = "text"
+ self.lm_config = lm_config
+ instruction = json.load(open(self.instruction_path))
+ instruction["examples"] = [tuple(e) for e in instruction["examples"]]
+ self.instruction: Instruction = instruction
+ self.tokenizer = tokenizer
+
+ def get_lm_api_input(
+ self, intro: str, examples: list[tuple[str, str]], current: str
+ ) -> APIInput:
+
+ """Return the require format for an API"""
+ message: list[dict[str, str]] | str
+ if "openai" in self.lm_config.provider:
+ if self.lm_config.mode == "chat":
+ message = [{"role": "system", "content": intro}]
+ for (x, y) in examples:
+ message.append(
+ {
+ "role": "system",
+ "name": "example_user",
+ "content": x,
+ }
+ )
+ message.append(
+ {
+ "role": "system",
+ "name": "example_assistant",
+ "content": y,
+ }
+ )
+ message.append({"role": "user", "content": current})
+ return message
+ elif self.lm_config.mode == "completion":
+ message = f"{intro}\n\n"
+ message += "Here are a few examples:\n"
+ for example in examples:
+ message += f"Observation\n:{example[0]}\n\n"
+ message += f"Action: {example[1]}\n\n"
+ message += "Now make prediction given the observation\n\n"
+ message += f"Observation\n:{current}\n\n"
+ message += "Action:"
+ return message
+ else:
+ raise ValueError(
+ f"OpenAI models do not support mode {self.lm_config.mode}"
+ )
+ elif "huggingface" in self.lm_config.provider:
+ # https://huggingface.co/blog/llama2#how-to-prompt-llama-2
+ # https://github.com/facebookresearch/llama/blob/main/llama/generation.py#L320
+ if "Llama-2" in self.lm_config.model:
+ if self.lm_config.mode == "chat":
+ B_INST, E_INST = "[INST]", "[/INST]"
+ B_SYS, E_SYS = "<>\n", "\n< >\n\n"
+ BOS, EOS = "", ""
+ # adding the system message to be the starting of the first example
+ examples = [
+ (
+ B_SYS + intro + E_SYS + examples[0][0],
+ examples[0][1],
+ )
+ ] + examples[1:]
+ message = "".join(
+ [
+ f"{BOS}{B_INST} {x.strip()} {E_INST} {y.strip()} {EOS}"
+ for (x, y) in examples
+ ]
+ )
+ # add the current observation
+ message += f"{BOS}{B_INST} {current.strip()} {E_INST} {self.instruction['meta_data'].get('force_prefix', '')}"
+
+ return message
+ else:
+ raise ValueError("Only chat mode is supported for Llama-2")
+ else:
+ raise ValueError(
+ f"Huggingface models do not support model_tag {self.lm_config.gen_config['model_tag']}"
+ )
+ elif "ours" in self.lm_config.provider:
+ message = f"{intro}\n\n"
+ message += "Now make prediction given the observation\n\n"
+ message += f"Observation\n:{current}\n\n"
+ message += "Action:"
+ return message
+ else:
+ raise NotImplementedError(
+ f"Provider {self.lm_config.provider} not implemented"
+ )
+
+ def construct(
+ self,
+ trajectory: Trajectory,
+ intent: str,
+ meta_data: dict[str, Any] = {},
+ ) -> APIInput:
+ raise NotImplementedError
+
+ def map_url_to_real(self, url: str) -> str:
+ """Map the urls to their real world counterparts"""
+ for i, j in URL_MAPPINGS.items():
+ if i in url:
+ url = url.replace(i, j)
+ return url
+
+ def map_url_to_local(self, url: str) -> str:
+ """Map the urls to their local counterparts"""
+ for i, j in URL_MAPPINGS.items():
+ if j in url:
+ url = url.replace(j, i)
+ # https
+ if j.replace("http", "https") in url:
+ url = url.replace(j.replace("http", "https"), i)
+ return url
+
+ def _extract_action(self, response: str) -> str:
+ raise NotImplementedError
+
+ def extract_action(self, response: str) -> str:
+ response = self._extract_action(response)
+ response = self.map_url_to_local(response)
+ return response
+
+
+class DirectPromptConstructor(PromptConstructor):
+ """The agent will direct predict the action"""
+
+ def __init__(
+ self,
+ instruction_path: str | Path,
+ lm_config: lm_config.LMConfig,
+ tokenizer: Tokenizer,
+ ):
+ super().__init__(instruction_path, lm_config, tokenizer)
+
+ def construct(
+ self,
+ trajectory: Trajectory,
+ intent: str,
+ meta_data: dict[str, Any] = {},
+ ) -> APIInput:
+ """Construct prompt given the trajectory"""
+ intro = self.instruction["intro"]
+ examples = self.instruction["examples"]
+ template = self.instruction["template"]
+ keywords = self.instruction["meta_data"]["keywords"]
+ state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
+
+ obs = state_info["observation"][self.obs_modality]
+ max_obs_length = self.lm_config.gen_config["max_obs_length"]
+ if max_obs_length:
+ obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
+
+ page = state_info["info"]["page"]
+ url = page.url
+ previous_action_str = meta_data["action_history"][-1]
+
+ # input x
+ current = template.format(
+ objective=intent,
+ url=self.map_url_to_real(url),
+ observation=obs,
+ previous_action=previous_action_str,
+ )
+
+ # make sure all keywords are replaced
+ assert all([f"{{k}}" not in current for k in keywords])
+ prompt = self.get_lm_api_input(intro, examples, current)
+ return prompt
+
+ def _extract_action(self, response: str) -> str:
+ action_splitter = self.instruction["meta_data"]["action_splitter"]
+ pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
+ match = re.search(pattern, response)
+ if match:
+ return match.group(1).strip()
+ else:
+ raise ActionParsingError(
+ f"Cannot parse action from response {response}"
+ )
+
+
+class CoTPromptConstructor(PromptConstructor):
+ """The agent will perform step-by-step reasoning before the answer"""
+
+ def __init__(
+ self,
+ instruction_path: str | Path,
+ lm_config: lm_config.LMConfig,
+ tokenizer: Tokenizer,
+ ):
+ super().__init__(instruction_path, lm_config, tokenizer)
+ self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
+
+ def construct(
+ self,
+ trajectory: Trajectory,
+ intent: str,
+ meta_data: dict[str, Any] = {},
+ ) -> APIInput:
+ intro = self.instruction["intro"]
+ examples = self.instruction["examples"]
+ template = self.instruction["template"]
+ keywords = self.instruction["meta_data"]["keywords"]
+ state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
+
+ obs = state_info["observation"][self.obs_modality]
+ max_obs_length = self.lm_config.gen_config["max_obs_length"]
+ if max_obs_length:
+ obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
+
+ page = state_info["info"]["page"]
+ url = page.url
+ previous_action_str = meta_data["action_history"][-1]
+ current = template.format(
+ objective=intent,
+ url=self.map_url_to_real(url),
+ observation=obs,
+ previous_action=previous_action_str,
+ )
+
+ assert all([f"{{k}}" not in current for k in keywords])
+
+ prompt = self.get_lm_api_input(intro, examples, current)
+ return prompt
+
+ def _extract_action(self, response: str) -> str:
+ # find the first occurence of action
+ action_splitter = self.instruction["meta_data"]["action_splitter"]
+ pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
+ match = re.search(pattern, response)
+ if match:
+ return match.group(1).strip()
+ else:
+ raise ActionParsingError(
+ f'Cannot find the answer phrase "{self.answer_phrase}" in "{response}"'
+ )
+
+class MyPromptConstructor(PromptConstructor):
+ """The agent will perform step-by-step reasoning before the answer"""
+ operation = [
+ r"#?(Click)#?\s*([0-9]+)",
+ r"#?(Type)#?\s*([0-9]+)\s+[\'\"]{0,1}([\s\S]+)[\'\"]{0,1}",
+ r"#?(Select)#?\s*([0-9]+)\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
+ r"#?(Scroll_up)#?",
+ r"#?(Scroll_down)#?",
+ r"#?(Goto)#?\s*(https?:\/\/[-a-z0-9]+(?:\.[-a-z0-9]+)*\.(?:com|cn|edu|uk)(?:\/[-a-z0-9_:@&?=+,.!/~*'%$]*)?)",
+ r"#?(Go_backward)#?",
+ r"#?(Go_forward)#?",
+ r"#?(Hover)#?\s*([0-9]+)",
+ r"#?(Answer)#?\s+(.+)",
+ r"#?(Login)#?",
+ r"#?(Verify)#?",
+ r"#?(Exit)#?",
+ r"#?(Record)#?\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
+ ]
+
+ translate = [
+ "click",
+ "type",
+ "select",
+ "scroll [up]",
+ "scroll [down]",
+ "goto",
+ "go_back",
+ "go_forward",
+ "hover",
+ "stop",
+ "stop",
+ "stop",
+ "stop",
+ "record",
+ ]
+
+ def __init__(
+ self,
+ instruction_path: str | Path,
+ lm_config: lm_config.LMConfig,
+ tokenizer: Tokenizer,
+ ):
+ super().__init__(instruction_path, lm_config, tokenizer)
+ self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
+ self.state = {}
+
+ def construct(
+ self,
+ trajectory: Trajectory,
+ intent: str,
+ meta_data: dict[str, Any] = {},
+ ) -> APIInput:
+ intro = self.instruction["intro"]
+ examples = self.instruction["examples"]
+ template = self.instruction["template"]
+ keywords = self.instruction["meta_data"]["keywords"]
+ finale = self.instruction["finale"]
+ state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
+
+ obs = state_info["observation"][self.obs_modality]
+ max_obs_length = self.lm_config.gen_config["max_obs_length"]
+ if max_obs_length:
+ obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
+
+ info = state_info["info"]
+ obs_metadata = info["observation_metadata"]["text"]
+ nodes = obs_metadata["obs_nodes_info"]
+ position_info = obs_metadata["position_info"]
+ html_parser = obs_metadata["html_parser"]
+ self.nodes = nodes
+
+ page = info["page"]
+ url = self.map_url_to_real(page.url)
+ position_bar = self._get_position_bar(position_info)
+
+ history = [f"{ix}. {his}" for his in meta_data["action_history"]]
+ if len(history) == 1:
+ previous_action_str = "None"
+ else:
+ previous_action_str = '\n'.join(history[1:])
+
+ self.state.update({
+ "url": url,
+ "html": obs,
+ "html_parser": html_parser,
+ "segment": "None",
+ "operation": "None",
+ })
+
+ current = template.format(
+ objective=intent,
+ url=url,
+ html=obs,
+ position=position_bar,
+ previous_action=previous_action_str,
+ )
+
+ assert all([f"{{k}}" not in current for k in keywords])
+
+ # prompt = self.get_lm_api_input(intro, examples, current)
+ prompt = current + finale
+
+ return prompt
+
+ def _extract_action(self, response: str) -> str:
+ # find the first occurence of action
+ self.state["intention"] = self._extract_intention(response)
+
+ for regex, act in zip(self.operation, self.translate):
+ match = re.search(regex, response)
+
+ if match:
+ m = match.groups()
+ if isinstance(m, tuple):
+ exact_act = m[0]
+ param = m[1:]
+ else:
+ exact_act = m
+ param = []
+
+ param = list(param)
+ if act in ['click', 'hover', 'type', 'select']:
+ if len(param) == 0:
+ continue
+
+ for node_id, node in self.nodes.items():
+ if node['label'] == param[0]:
+ label = param[0]
+ hp = self.state["html_parser"]
+ bid = hp.id_label_converter(label)
+ segment = hp.get_segment(bid)
+
+ print('[Label]', label, bid, segment)
+ self.state["segment"] = segment
+ #self._extract_segment(self.state["html"], label)
+ if act not in ['select']:
+ param[0] = node_id
+ break
+
+
+ if act in ['stop', 'select', 'record']:
+ if len(param) > 0:
+ param[-1] = param[-1].strip("\'\"")
+
+ if act in ['type']:
+ print('In prompt constructer', param[-1])
+ if len(param) > 0:
+ param[-1] = param[-1].strip("\'\"")
+ print(param[-1])
+ if param[-1].endswith('\n'):
+ param[-1] = param[-1][:-1]
+ param.append('1')
+ else:
+ param.append('0')
+
+ command = act
+ for p in param:
+ command += f" [{p}]"
+
+ print(command)
+ return command
+
+ raise ActionParsingError(
+ f'Cannot find the answer phrase in "{response}"'
+ )
+
+ @staticmethod
+ def _get_position_bar(data):
+ position = data.get("position", 0.0)
+ page_height = data.get("page_height", 1.0)
+ left_bar = '-' * int(position)
+ right_bar = '-' * int(max(1, page_height - position))
+ return f'[0{left_bar}|{round(position, 1)}{right_bar}{round(page_height, 1)}]'
+
+ @staticmethod
+ def _extract_intention(response, lang='en'):
+ if lang == 'en':
+ matches = re.findall(r"#Thinking Process:\s*(.+)\s*#Operation:", response)
+ print('[Try to match]', matches)
+ else:
+ matches = re.findall(r"#思考过程: (.+)", response)
+
+ if matches:
+ return matches[-1]
+ else:
+ return None
+
+ @staticmethod
+ def _extract_segment(html: str, tag: str):
+ tag = f'[{tag}]'
+ has_content = False
+
+ def _left(html, start):
+ nonlocal has_content
+ left_cnt, right_cnt = 0, 0
+ for i in range(start, -1, -1):
+ if html[i] == '<':
+ left_cnt += 1
+ elif html[i] == '>':
+ if html[i - 2] != '|' and html[i - 2] != '>':
+ has_content = True
+ right_cnt += 1
+ elif html[i] == '|':
+ if html[i + 2] != '<' and html[i + 2] != '>':
+ has_content = True
+ if left_cnt == right_cnt + 1:
+ return i
+ return -1
+
+ def _right(html, start):
+ nonlocal has_content
+ left_cnt, right_cnt = 0, 0
+ for i in range(start, len(html), 1):
+ if html[i] == '<':
+ left_cnt += 1
+ elif html[i] == '>':
+ if html[i - 2] != '|' and html[i - 2] != '>':
+ has_content = True
+ right_cnt += 1
+ elif html[i] == '|':
+ if html[i + 2] != '<' and html[i + 2] != '>':
+ has_content = True
+ if left_cnt + 1 == right_cnt:
+ return i + 1
+ return -1
+
+ tag_start = html.find(tag)
+
+ if tag_start == -1:
+ return None
+
+ left_bound, right_bound = _left(html, tag_start), _right(html, tag_start)
+ while True:
+ if left_bound == -1 or right_bound == -1:
+ return None
+
+ if has_content:
+ break
+
+ else:
+ lb, rb = _left(html, left_bound - 1), _right(html, right_bound + 1)
+ if lb == -1 or rb == -1:
+ break
+ if rb - lb > 150:
+ break
+ else:
+ left_bound, right_bound = lb, rb
+
+ segment = html[left_bound:right_bound]
+
+ if len(segment) > 150:
+ return segment[:150] + '...>'
+
+ return segment
+
+class NewASPromptConstructor(PromptConstructor):
+ """The agent will perform step-by-step reasoning before the answer"""
+ operation = [
+ r"(click)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
+ r"(type_string)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*,\s*(True|False)\s*\)",
+ r"(select)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*\)",
+ r"(scroll_page)\(\s*[\'\"]up[\'\"]\s*\)",
+ r"(scroll_page)\(\s*[\'\"]down[\'\"]\s*\)",
+ r"(jump_to)\(\s*[\'\"](.+)[\'\"]\s*,\s*(True|False)\s*\)",
+ r"(go)\(\s*[\'\"]backward[\'\"]\s*\)",
+ r"(go)\(\s*[\'\"]forward[\'\"]\s*\)",
+ r"(hover)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
+ r"(finish)\(\s*\)",
+ r"(finish)\(\s*(.+)\s*\)",
+ r"(record)\(\s*[\'\"](.+)[\'\"]\s*\)",
+ r"(switch_tab)\([\d]+\)"
+ ]
+
+ translate = [
+ "click",
+ "type",
+ "select",
+ "scroll [up]",
+ "scroll [down]",
+ "goto",
+ "go_back",
+ "go_forward",
+ "hover",
+ "stop",
+ "stop",
+ "record",
+ "page_focus",
+ ]
+
+ def __init__(
+ self,
+ instruction_path: str | Path,
+ lm_config: lm_config.LMConfig,
+ tokenizer: Tokenizer,
+ ):
+ super().__init__(instruction_path, lm_config, tokenizer)
+ self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
+ self.state = {}
+
+ def construct(
+ self,
+ trajectory: Trajectory,
+ intent: str,
+ meta_data: dict[str, Any] = {},
+ ) -> APIInput:
+ intro = self.instruction["intro"]
+ examples = self.instruction["examples"]
+ template = self.instruction["template"]
+ keywords = self.instruction["meta_data"]["keywords"]
+ finale = self.instruction["finale"]
+ state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
+
+ obs = state_info["observation"][self.obs_modality]
+ max_obs_length = self.lm_config.gen_config["max_obs_length"]
+ if max_obs_length:
+ obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
+
+ info = state_info["info"]
+ obs_metadata = info["observation_metadata"]["text"]
+ nodes = obs_metadata["obs_nodes_info"]
+ position_info = obs_metadata["position_info"]
+ html_parser = obs_metadata["html_parser"]
+ tabs_str = obs_metadata["tab_title"]
+ self.nodes = nodes
+
+ page = info["page"]
+ url = self.map_url_to_real(page.url)
+ position_bar = self._get_position_bar(position_info)
+
+ history = meta_data["action_history"]
+ if len(history) == 1:
+ previous_action_str = "None"
+ else:
+ previous_action_str = '\n'.join(history[1:])
+
+ self.state.update({
+ "url": url,
+ "html": obs,
+ "html_parser": html_parser,
+ "segment": "None",
+ "operation": "None",
+ })
+
+ current = template.format(
+ objective=intent,
+ url=url,
+ html=obs,
+ position=position_bar,
+ previous_action=previous_action_str,
+ tabs=tabs_str,
+ )
+
+ assert all([f"{{k}}" not in current for k in keywords])
+
+ # prompt = self.get_lm_api_input(intro, examples, current)
+ prompt = current + finale
+
+ return prompt
+
+ def _extract_action(self, response: str) -> str:
+ # find the first occurence of action
+ # self.state["intention"] = self._extract_intention(response)
+
+ for regex, act in zip(self.operation, self.translate):
+ match = re.search(regex, response)
+ if match:
+ m = match.groups()
+ if isinstance(m, tuple):
+ exact_act = m[0]
+ param = m[1:]
+ else:
+ exact_act = m
+ param = []
+
+ print(exact_act, param)
+ param = list(param)
+ if act in ['click', 'hover', 'type', 'select']:
+ if len(param) == 0:
+ continue
+
+ for node_id, node in self.nodes.items():
+ if node['label'] == param[0]:
+ label = param[0]
+ hp = self.state["html_parser"]
+ bid = hp.id_label_converter(label)
+ segment = hp.get_segment(bid)
+
+ print('[Label]', label, bid, segment)
+ self.state["segment"] = segment
+ #self._extract_segment(self.state["html"], label)
+ if act not in ['select']:
+ param[0] = node_id
+ break
+
+ if len(param) > 0:
+ if act in ['stop', 'select', 'record']:
+ param[-1] = param[-1].strip("\'\"")
+ if act in ['type', 'goto']:
+ param[-1] = '1' if param[-1] == 'True' else '0'
+
+ command = act
+ for p in param:
+ command += f" [{p}]"
+
+ print(command)
+ return command
+
+ raise ActionParsingError(
+ f'Cannot find the answer phrase in "{response}"'
+ )
+
+ @staticmethod
+ def _get_position_bar(data):
+ position = data.get("position", 0.0)
+ page_height = data.get("page_height", 1.0)
+ return f"{round(position, 1)} / {round(page_height, 1)}"
\ No newline at end of file
diff --git a/webarena/agent/prompts/raw/new_action_prompt.py b/webarena/agent/prompts/raw/new_action_prompt.py
new file mode 100644
index 0000000..5382b03
--- /dev/null
+++ b/webarena/agent/prompts/raw/new_action_prompt.py
@@ -0,0 +1,118 @@
+prompt = {
+ "intro": "",
+ "examples": [],
+ "template": """ {html}
+
+You are a helpful assistant that can assist with web navigation tasks.
+You are given a simplified html webpage and a task description.
+Your goal is to complete the task. You can use the provided functions below to interact with the current webpage.
+
+#Provided functions:
+def click(element_id: str) -> None:
+ \"\"\"
+ Click on the element with the specified id.
+
+ Args:
+ element_id: The id of the element.
+ \"\"\"
+
+def hover(element_id: str) -> None:
+ \"\"\"
+ Hover on the element with the specified id.
+
+ Args:
+ element_id: The id of the element.
+ \"\"\"
+
+def select(element_id: str, option: str) -> None:
+ \"\"\"
+ Select an option from a dropdown.
+
+ Args:
+ element_id: The id of the element.
+ option: Value of the option to select.
+ \"\"\"
+
+def type_string(element_id: str, content: str, press_enter: bool) -> None:
+ \"\"\"
+ Type a string into the element with the specified id.
+
+ Args:
+ element_id: The id of the element.
+ content: The string to type.
+ press_enter: Whether to press enter after typing the string.
+ \"\"\"
+
+def scroll_page(direction: Literal['up', 'down']) -> None:
+ \"\"\"
+ Scroll down/up one page.
+
+ Args:
+ direction: The direction to scroll.
+ \"\"\"
+
+def go(direction: Literal['forward', 'backward']) -> None:
+ \"\"\"
+ Go forward/backward
+
+ Args:
+ direction: The direction to go to.
+ \"\"\"
+
+def jump_to(url: str, new_tab: bool) -> None:
+ \"\"\"
+ Jump to the specified url.
+
+ Args:
+ url: The url to jump to.
+ new_tab: Whether to open the url in a new tab.
+ \"\"\"
+
+def switch_tab(tab_index: int) -> None:
+ \"\"\"
+ Switch to the specified tab.
+
+ Args:
+ tab_index: The index of the tab to switch to.
+ \"\"\"
+
+def user_input(message: str) -> str:
+ \"\"\"
+ Wait for user input.
+
+ Args:
+ message: The message to display to the user.
+
+ Returns: The user input.
+ \"\"\"
+
+def finish(answer: Optional[str]) -> None:
+ \"\"\"
+ Finish the task (optionally with an answer).
+
+ Args:
+ answer: The answer to the task.
+ \"\"\"
+
+#Previous commands: {previous_action}
+
+#Window tabs: {tabs}
+
+#Current viewport (pages): {position}
+
+#Task: {objective}
+
+You should output one command to interact to the currrent webpage.
+You should add a brief comment to your command to explain your reasoning and thinking process.
+""",
+ "finale": "",
+ "meta_data": {
+ "observation": "html",
+ "action_type": "id_html_nasc_tree",
+ "keywords": ["url", "html", "objective", "position", "previous_action", "tabs"],
+ "prompt_constructor": "NewASPromptConstructor",
+ "answer_phrase": "",
+ "action_splitter": "#"
+ },
+}
+
diff --git a/webarena/agent/prompts/raw/p_cot_id_actree_2s.py b/webarena/agent/prompts/raw/p_cot_id_actree_2s.py
new file mode 100644
index 0000000..b85e54c
--- /dev/null
+++ b/webarena/agent/prompts/raw/p_cot_id_actree_2s.py
@@ -0,0 +1,82 @@
+prompt = {
+ "intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
+
+Here's the information you'll have:
+The user's objective: This is the task you're trying to complete.
+The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
+The current web page's URL: This is the page you're currently navigating.
+The open tabs: These are the tabs you have open.
+The previous action: This is the action you just performed. It may be helpful to track your progress.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Tab Management Actions:
+`new_tab`: Open a new, empty browser tab.
+`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
+`close_tab`: Close the currently active tab.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Homepage:
+If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
+http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
+
+To be successful, it is very important to follow the following rules:
+1. You should only issue an action that is valid given the current observation
+2. You should only issue one action at a time.
+3. You should follow the examples to reason step by step and then issue the next action.
+4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
+5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
+ "examples": [
+ (
+ """OBSERVATION:
+[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
+ [1749] StaticText '$279.49'
+ [1757] button 'Add to Cart'
+ [1760] button 'Add to Wish List'
+ [1761] button 'Add to Compare'
+URL: http://onestopmarket.com/office-products/office-electronics.html
+OBJECTIVE: What is the price of HP Inkjet Fax Machine
+PREVIOUS ACTION: None""",
+ "Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
+ ),
+ (
+ """OBSERVATION:
+[164] textbox 'Search' focused: True required: False
+[171] button 'Go'
+[174] link 'Find directions between two points'
+[212] heading 'Search Results'
+[216] button 'Close'
+URL: http://openstreetmap.org
+OBJECTIVE: Show me the restaurants near CMU
+PREVIOUS ACTION: None""",
+ "Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
+ ),
+ ],
+ "template": """OBSERVATION:
+{observation}
+URL: {url}
+OBJECTIVE: {objective}
+PREVIOUS ACTION: {previous_action}""",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": ["url", "objective", "observation", "previous_action"],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ },
+}
diff --git a/webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py b/webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py
new file mode 100644
index 0000000..945cd95
--- /dev/null
+++ b/webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py
@@ -0,0 +1,82 @@
+prompt = {
+ "intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
+
+Here's the information you'll have:
+The user's objective: This is the task you're trying to complete.
+The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
+The current web page's URL: This is the page you're currently navigating.
+The open tabs: These are the tabs you have open.
+The previous action: This is the action you just performed. It may be helpful to track your progress.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Tab Management Actions:
+`new_tab`: Open a new, empty browser tab.
+`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
+`close_tab`: Close the currently active tab.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
+
+Homepage:
+If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
+http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
+
+To be successful, it is very important to follow the following rules:
+1. You should only issue an action that is valid given the current observation
+2. You should only issue one action at a time.
+3. You should follow the examples to reason step by step and then issue the next action.
+4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
+5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
+ "examples": [
+ (
+ """OBSERVATION:
+[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
+ [1749] StaticText '$279.49'
+ [1757] button 'Add to Cart'
+ [1760] button 'Add to Wish List'
+ [1761] button 'Add to Compare'
+URL: http://onestopmarket.com/office-products/office-electronics.html
+OBJECTIVE: What is the price of HP Inkjet Fax Machine
+PREVIOUS ACTION: None""",
+ "Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
+ ),
+ (
+ """OBSERVATION:
+[164] textbox 'Search' focused: True required: False
+[171] button 'Go'
+[174] link 'Find directions between two points'
+[212] heading 'Search Results'
+[216] button 'Close'
+URL: http://openstreetmap.org
+OBJECTIVE: Show me the restaurants near CMU
+PREVIOUS ACTION: None""",
+ "Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
+ ),
+ ],
+ "template": """OBSERVATION:
+{observation}
+URL: {url}
+OBJECTIVE: {objective}
+PREVIOUS ACTION: {previous_action}""",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": ["url", "objective", "observation", "previous_action"],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ },
+}
diff --git a/webarena/agent/prompts/raw/p_direct_id_actree_2s.py b/webarena/agent/prompts/raw/p_direct_id_actree_2s.py
new file mode 100644
index 0000000..8d4e4f6
--- /dev/null
+++ b/webarena/agent/prompts/raw/p_direct_id_actree_2s.py
@@ -0,0 +1,80 @@
+prompt = {
+ "intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
+
+Here's the information you'll have:
+The user's objective: This is the task you're trying to complete.
+The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
+The current web page's URL: This is the page you're currently navigating.
+The open tabs: These are the tabs you have open.
+The previous action: This is the action you just performed. It may be helpful to track your progress.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Tab Management Actions:
+`new_tab`: Open a new, empty browser tab.
+`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
+`close_tab`: Close the currently active tab.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
+
+Homepage:
+If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
+http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
+
+To be successful, it is very important to follow the following rules:
+1. You should only issue an action that is valid given the current observation
+2. You should only issue one action at a time.
+3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.
+5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
+ "examples": [
+ (
+ """OBSERVATION:
+[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
+ [1749] StaticText '$279.49'
+ [1757] button 'Add to Cart'
+ [1760] button 'Add to Wish List'
+ [1761] button 'Add to Compare'
+URL: http://onestopmarket.com/office-products/office-electronics.html
+OBJECTIVE: What is the price of HP Inkjet Fax Machine
+PREVIOUS ACTION: None""",
+ "```stop [$279.49]```",
+ ),
+ (
+ """OBSERVATION:
+[164] textbox 'Search' focused: True required: False
+[171] button 'Go'
+[174] link 'Find directions between two points'
+[212] heading 'Search Results'
+[216] button 'Close'
+URL: http://openstreetmap.org
+OBJECTIVE: Show me the restaurants near CMU
+PREVIOUS ACTION: None""",
+ "```type [164] [restaurants near CMU] [1]```",
+ ),
+ ],
+ "template": """OBSERVATION:
+{observation}
+URL: {url}
+OBJECTIVE: {objective}
+PREVIOUS ACTION: {previous_action}""",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": ["url", "objective", "observation", "previous_action"],
+ "prompt_constructor": "DirectPromptConstructor",
+ "action_splitter": "```"
+ },
+}
diff --git a/webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py b/webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py
new file mode 100644
index 0000000..c399454
--- /dev/null
+++ b/webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py
@@ -0,0 +1,81 @@
+prompt = {
+ "intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
+
+Here's the information you'll have:
+The user's objective: This is the task you're trying to complete.
+The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
+The current web page's URL: This is the page you're currently navigating.
+The open tabs: These are the tabs you have open.
+The previous action: This is the action you just performed. It may be helpful to track your progress.
+
+The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Tab Management Actions:
+`new_tab`: Open a new, empty browser tab.
+`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
+`close_tab`: Close the currently active tab.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
+
+Homepage:
+If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
+http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
+
+To be successful, it is very important to follow the following rules:
+1. You should only issue an action that is valid given the current observation
+2. You should only issue one action at a time.
+4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```".
+5. Issue stop action when you think you have achieved the objective.""",
+ "examples": [
+ (
+ """OBSERVATION:
+[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
+ [1749] StaticText '$279.49'
+ [1757] button 'Add to Cart'
+ [1760] button 'Add to Wish List'
+ [1761] button 'Add to Compare'
+URL: http://onestopmarket.com/office-products/office-electronics.html
+OBJECTIVE: What is the price of HP Inkjet Fax Machine
+PREVIOUS ACTION: None""",
+ "```stop [$279.49]```",
+ ),
+ (
+ """OBSERVATION:
+[164] textbox 'Search' focused: True required: False
+[171] button 'Go'
+[174] link 'Find directions between two points'
+[212] heading 'Search Results'
+[216] button 'Close'
+URL: http://openstreetmap.org
+OBJECTIVE: Show me the restaurants near CMU
+PREVIOUS ACTION: None""",
+ "```type [164] [restaurants near CMU] [1]```",
+ ),
+ ],
+ "template": """OBSERVATION:
+{observation}
+URL: {url}
+OBJECTIVE: {objective}
+PREVIOUS ACTION: {previous_action}""",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": ["url", "objective", "observation", "previous_action"],
+ "prompt_constructor": "CoTPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```"
+ },
+}
diff --git a/webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py b/webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py
new file mode 100644
index 0000000..6278d2b
--- /dev/null
+++ b/webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py
@@ -0,0 +1,83 @@
+prompt = {
+ "intro": """You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:
+
+Page Operation Actions:
+`click [id]`: This action clicks on an element with a specific id on the webpage.
+`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
+`hover [id]`: Hover over an element with id.
+`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
+`scroll [direction=down|up]`: Scroll the page up or down.
+
+Tab Management Actions:
+`new_tab`: Open a new, empty browser tab.
+`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
+`close_tab`: Close the currently active tab.
+
+URL Navigation Actions:
+`goto [url]`: Navigate to a specific URL.
+`go_back`: Navigate to the previously viewed page.
+`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
+
+Completion Action:
+`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
+
+Homepage:
+If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
+
+You can only issue one action at a time""",
+
+ "examples": [
+ (
+ """Observation:
+[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
+ [1749] StaticText '$279.49'
+ [1757] button 'Add to Cart'
+ [1760] button 'Add to Wish List'
+ [1761] button 'Add to Compare'
+URL: http://onestopmarket.com/office-products/office-electronics.html
+Objective: What is the price of HP Inkjet Fax Machine
+Previous action: None""",
+ "```stop [$279.49]```",
+ ),
+ (
+ """Observation:
+[164] textbox 'Search' focused: True required: False
+[171] button 'Go'
+[174] link 'Find directions between two points'
+[212] heading 'Search Results'
+[216] button 'Close'
+URL: http://openstreetmap.org
+Objective: Show me the restaurants near CMU
+Previous action: None""",
+ "```type [164] [restaurants near CMU] [1]```",
+ ),
+ (
+ """Observation:
+[2036] button 'Sort by: New' hasPopup: menu expanded: False
+ [587] link 'US Marine’s adoption of Afghan war orphan voided'
+ [989] time 'March 30, 2023 at 15:03:48 AM UTC'
+ [602] link 'York student uses AI chatbot to get parking fine revoked'
+ [1025] time 'March 15, 2023 at 7:48:34 AM UTC'
+ [617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'
+ [1025] time 'March 2, 2023 at 3:46:01 AM UTC'
+URL: http://reddit.com/f/news/new
+Objective: Open the most recent post that was published prior to March 1st.
+Previous action: None""",
+ "```scroll [down]```",
+ )
+ ],
+ "template": """Observation:
+{observation}
+URL: {url}
+Objective: {objective}
+Previous action: {previous_action}""",
+ "meta_data": {
+ "observation": "accessibility_tree",
+ "action_type": "id_accessibility_tree",
+ "keywords": ["url", "objective", "observation", "previous_action"],
+ "prompt_constructor": "DirectPromptConstructor",
+ "answer_phrase": "In summary, the next action I will perform is",
+ "action_splitter": "```",
+ "force_prefix": "```"
+ },
+}
diff --git a/webarena/agent/prompts/raw/test_prompt.py b/webarena/agent/prompts/raw/test_prompt.py
new file mode 100644
index 0000000..5979e58
--- /dev/null
+++ b/webarena/agent/prompts/raw/test_prompt.py
@@ -0,0 +1,41 @@
+prompt = {
+ "intro": "",
+ "examples": [],
+ "template": """ {html}
+
+You are a helpful assistant that can assist with web navigation tasks.
+You are given a simplified html webpage and a task description.
+Your goal is to complete the task. You can perform the specified operations below to interact with the webpage.
+
+#Valid operations: - #Click# id: Click on the element with the specified id
+- #Scroll_up#: Scroll up 1 page.
+- #Scroll_down#: Scroll down 1 page.
+- #Go_backward#: Go back to the previous page.
+- #Go_forward#: Go forward to the next page.
+- #Hover# id: Hover over the element with the specified id.
+- #Type# id "text": Type in the text at the element with the specified id.
+- #Select# id "option": Select the option at the element with the specified id.
+- #Record# "content": Mark content that is useful in answering the question.
+- #Answer# "text": output the text as the answer to the user.
+- #Exit#: Complete the task and exit the program.
+
+#Current viewport position: {position}
+
+#Previous Operation: {previous_action}
+
+#Task: {objective}
+""",
+ "finale": """
+Your output SHOULD be in the following format:
+#Operation: {Next operation to perform}
+""",
+ "meta_data": {
+ "observation": "html",
+ "action_type": "id_html_tree",
+ "keywords": ["url", "html", "objective", "position", "previous_action"],
+ "prompt_constructor": "MyPromptConstructor",
+ "answer_phrase": "",
+ "action_splitter": "#"
+ },
+}
+
diff --git a/webarena/agent/prompts/to_json.py b/webarena/agent/prompts/to_json.py
new file mode 100644
index 0000000..efb283c
--- /dev/null
+++ b/webarena/agent/prompts/to_json.py
@@ -0,0 +1,26 @@
+import glob
+import importlib
+import json
+import os
+
+
+# use the current directory as the root
+def run() -> None:
+ """Convert all python files in agent/prompts to json files in agent/prompts/jsons
+
+ Python files are easiser to edit
+ """
+ for p_file in glob.glob(f"agent/prompts/raw/*.py"):
+ # import the file as a module
+ base_name = os.path.basename(p_file).replace(".py", "")
+ module = importlib.import_module(f"agent.prompts.raw.{base_name}")
+ prompt = module.prompt
+ # save the prompt as a json file
+ os.makedirs("agent/prompts/jsons", exist_ok=True)
+ with open(f"agent/prompts/jsons/{base_name}.json", "w+") as f:
+ json.dump(prompt, f, indent=2)
+ print(f"Done convert python files to json")
+
+
+if __name__ == "__main__":
+ run()
diff --git a/webarena_replication.py b/webarena_replication.py
new file mode 100644
index 0000000..8c4a542
--- /dev/null
+++ b/webarena_replication.py
@@ -0,0 +1,464 @@
+"""Script to run end-to-end evaluation on the benchmark"""
+import argparse
+import glob
+import json
+import logging
+import os
+import random
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+import openai
+
+from webarena.agent import (
+ Agent,
+ PromptAgent,
+ TeacherForcingAgent,
+ construct_agent,
+)
+from webarena.agent.prompts import *
+from browser_env import (
+ Action,
+ ActionTypes,
+ ScriptBrowserEnv,
+ StateInfo,
+ Trajectory,
+ create_stop_action,
+)
+from browser_env.actions import is_equivalent
+from browser_env.auto_login import get_site_comb_from_filepath
+from browser_env.helper_functions import (
+ RenderHelper,
+ get_action_description,
+)
+from evaluation_harness import evaluator_router
+from tqdm import tqdm
+import nltk
+nltk.download('punkt_tab')
+
+LOG_FOLDER = "log_files"
+Path(LOG_FOLDER).mkdir(parents=True, exist_ok=True)
+LOG_FILE_NAME = f"{LOG_FOLDER}/log_{time.strftime('%Y%m%d%H%M%S', time.localtime())}_{random.randint(0, 10000)}.log"
+
+logger = logging.getLogger("logger")
+logger.setLevel(logging.INFO)
+
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.DEBUG)
+logger.addHandler(console_handler)
+
+file_handler = logging.FileHandler(LOG_FILE_NAME)
+file_handler.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+# Set the log format
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+console_handler.setFormatter(formatter)
+file_handler.setFormatter(formatter)
+
+
+def config() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Run end-to-end evaluation on the benchmark"
+ )
+ parser.add_argument(
+ "--render", action="store_true", help="Render the browser"
+ )
+ parser.add_argument(
+ "--slow_mo",
+ type=int,
+ default=0,
+ help="Slow down the browser by the specified amount",
+ )
+ parser.add_argument(
+ "--action_set_tag", default="id_accessibility_tree", help="Action type"
+ )
+ parser.add_argument(
+ "--observation_type",
+ choices=["accessibility_tree", "html", "image"],
+ default="accessibility_tree",
+ help="Observation type",
+ )
+ parser.add_argument(
+ "--current_viewport_only",
+ action="store_true",
+ help="Only use the current viewport for the observation",
+ )
+ parser.add_argument("--viewport_width", type=int, default=1280)
+ parser.add_argument("--viewport_height", type=int, default=720)
+ parser.add_argument("--save_trace_enabled", action="store_true")
+ parser.add_argument("--sleep_after_execution", type=float, default=0.0)
+
+ parser.add_argument("--max_steps", type=int, default=30)
+
+ # agent config
+ parser.add_argument("--agent_type", type=str, default="prompt")
+ parser.add_argument(
+ "--instruction_path",
+ type=str,
+ default="agents/prompts/state_action_agent.json",
+ )
+ parser.add_argument(
+ "--parsing_failure_th",
+ help="When concesecutive parsing failure exceeds this threshold, the agent will stop",
+ type=int,
+ default=3,
+ )
+ parser.add_argument(
+ "--repeating_action_failure_th",
+ help="When concesecutive repeating action exceeds this threshold, the agent will stop",
+ type=int,
+ default=3,
+ )
+
+ # lm config
+ parser.add_argument("--provider", type=str, default="openai")
+ parser.add_argument("--model", type=str, default="gpt-3.5-turbo-0613")
+ parser.add_argument("--mode", type=str, default="chat")
+ parser.add_argument("--temperature", type=float, default=1.0)
+ parser.add_argument("--top_p", type=float, default=0.9)
+ parser.add_argument("--context_length", type=int, default=0)
+ parser.add_argument("--max_tokens", type=int, default=384)
+ parser.add_argument("--stop_token", type=str, default=None)
+ parser.add_argument("--cuda", type=str, default='0')
+ parser.add_argument(
+ "--max_retry",
+ type=int,
+ help="max retry times to perform generations when parsing fails",
+ default=1,
+ )
+ parser.add_argument(
+ "--max_obs_length",
+ type=int,
+ help="when not zero, will truncate the observation to this length before feeding to the model",
+ default=1920,
+ )
+ parser.add_argument(
+ "--model_endpoint",
+ help="huggingface model endpoint",
+ type=str,
+ default="",
+ )
+
+ # example config
+ parser.add_argument("--test_start_idx", type=int, default=0)
+ parser.add_argument("--test_end_idx", type=int, default=1000)
+ parser.add_argument("--sample", type=int, default=1)
+
+ # logging related
+ parser.add_argument("--result_dir", type=str, default="")
+ args = parser.parse_args()
+
+ # check the whether the action space is compatible with the observation space
+ if (
+ args.action_set_tag == "id_accessibility_tree"
+ and args.observation_type not in ["accessibility_tree", "html"]
+ ):
+ raise ValueError(
+ f"Action type {args.action_set_tag} is incompatible with the observation type {args.observation_type}"
+ )
+
+ return args
+
+
+def early_stop(
+ trajectory: Trajectory, max_steps: int, thresholds: dict[str, int]
+) -> tuple[bool, str]:
+ """Check whether need to early stop"""
+
+ # reach the max step
+ num_steps = (len(trajectory) - 1) / 2
+ if num_steps >= max_steps:
+ return True, f"Reach max steps {max_steps}"
+
+ last_k_actions: list[Action]
+ action_seq: list[Action]
+
+ # Case: parsing failure for k times
+ k = thresholds["parsing_failure"]
+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]
+ if len(last_k_actions) >= k:
+ if all(
+ [
+ action["action_type"] == ActionTypes.NONE
+ for action in last_k_actions
+ ]
+ ):
+ return True, f"Failed to parse actions for {k} times"
+
+ # Case: same action for k times
+ k = thresholds["repeating_action"]
+ last_k_actions = trajectory[1::2][-k:] # type: ignore[assignment]
+ action_seq = trajectory[1::2] # type: ignore[assignment]
+
+ if len(action_seq) == 0:
+ return False, ""
+
+ last_action: Action = action_seq[-1]
+
+ if last_action["action_type"] != ActionTypes.TYPE:
+ if len(last_k_actions) >= k:
+ if all(
+ [
+ is_equivalent(action, last_action)
+ for action in last_k_actions
+ ]
+ ):
+ return True, f"Same action for {k} times"
+
+ else:
+ # check the action sequence
+ if (
+ sum([is_equivalent(action, last_action) for action in action_seq])
+ >= k
+ ):
+ return True, f"Same typing action for {k} times"
+
+ return False, ""
+
+
+def test(
+ args: argparse.Namespace,
+ agent: Agent | PromptAgent | TeacherForcingAgent,
+ config_file_list: list[str],
+) -> None:
+ scores = []
+ max_steps = args.max_steps
+
+ early_stop_thresholds = {
+ "parsing_failure": args.parsing_failure_th,
+ "repeating_action": args.repeating_action_failure_th,
+ }
+
+ env = ScriptBrowserEnv(
+ headless=not args.render,
+ slow_mo=args.slow_mo,
+ observation_type=args.observation_type,
+ current_viewport_only=args.current_viewport_only,
+ viewport_size={
+ "width": args.viewport_width,
+ "height": args.viewport_height,
+ },
+ save_trace_enabled=args.save_trace_enabled,
+ sleep_after_execution=args.sleep_after_execution,
+ )
+
+ for config_file in tqdm(config_file_list):
+ try:
+ render_helper = RenderHelper(
+ config_file, args.result_dir, args.action_set_tag
+ )
+
+ # get intent
+ with open(config_file) as f:
+ _c = json.load(f)
+ intent = _c["intent"]
+ task_id = _c["task_id"]
+ if task_id in list(range(600, 650))+list(range(681, 689)):
+ # continue
+ print("Reddit post task. Sleep 30 mins.")
+ time.sleep(1800)
+ # automatically login
+ if _c["storage_state"]:
+ cookie_file_name = os.path.basename(_c["storage_state"])
+ comb = get_site_comb_from_filepath(cookie_file_name)
+ temp_dir = tempfile.mkdtemp()
+ # subprocess to renew the cookie
+ subprocess.run(
+ [
+ "python",
+ "browser_env/auto_login.py",
+ "--auth_folder",
+ temp_dir,
+ "--site_list",
+ *comb,
+ ]
+ )
+ _c["storage_state"] = f"{temp_dir}/{cookie_file_name}"
+ assert os.path.exists(_c["storage_state"])
+ # update the config file
+ config_file = f"{temp_dir}/{os.path.basename(config_file)}"
+ with open(config_file, "w") as f:
+ json.dump(_c, f)
+
+ logger.info(f"[Config file]: {config_file}")
+ logger.info(f"[Intent]: {intent}")
+
+ agent.reset(config_file)
+ trajectory: Trajectory = []
+ obs, info = env.reset(options={"config_file": config_file})
+ obs["text"] = obs["text"][0]
+ state_info: StateInfo = {"observation": obs, "info": info}
+ trajectory.append(state_info)
+
+ meta_data = {"action_history": ["None"]}
+
+ trace = []
+ while True:
+ early_stop_flag, stop_info = early_stop(
+ trajectory, max_steps, early_stop_thresholds
+ )
+
+ if early_stop_flag:
+ action = create_stop_action(f"Early stop: {stop_info}")
+ else:
+ prompt = agent.prompt_constructor.construct(
+ trajectory, intent, meta_data
+ )
+ try:
+ action = agent.next_action(
+ trajectory, intent, meta_data=meta_data
+ )
+ except ValueError as e:
+ # get the error message
+ action = create_stop_action(f"ERROR: {str(e)}")
+
+ trajectory.append(action)
+
+ action_str = get_action_description(
+ action,
+ state_info["info"]["observation_metadata"],
+ action_set_tag=args.action_set_tag,
+ prompt_constructor=agent.prompt_constructor
+ if isinstance(agent, PromptAgent)
+ else None,
+ )
+ render_helper.render(
+ action, state_info, meta_data, args.render_screenshot
+ )
+ meta_data["action_history"].append(action_str)
+
+ trace.append({
+ "source": prompt,
+ "target": action_str.split(' #HTML Segment')[0],
+ })
+
+ if action["action_type"] == ActionTypes.STOP:
+ break
+
+ obs, _, terminated, _, info = env.step(action)
+ obs["text"] = obs["text"][0]
+ state_info = {"observation": obs, "info": info}
+ trajectory.append(state_info)
+
+ if terminated:
+ # add a action place holder
+ trajectory.append(create_stop_action(""))
+ break
+
+ evaluator = evaluator_router(config_file)
+ score = evaluator(
+ trajectory=trajectory,
+ config_file=config_file,
+ page=env.page,
+ client=env.get_page_client(env.page),
+ )
+
+ scores.append(score)
+
+ if score == 1:
+ logger.info(f"[Result] (PASS) {config_file}")
+ else:
+ logger.info(f"[Result] (FAIL) {config_file}")
+
+ if args.save_trace_enabled:
+ env.save_trace(
+ Path(args.result_dir) / "traces" / f"{task_id}.zip"
+ )
+ result = {
+ "id": task_id,
+ "score": score,
+ "trace": trace,
+ }
+ with open(Path(args.result_dir) / "traces" / f"trace_{task_id}.json", "w") as f:
+ json.dump(result, f, indent=4)
+
+ except openai.OpenAIError as e:
+ logger.info(f"[OpenAI Error] {repr(e)}")
+ except Exception as e:
+ logger.info(f"[Unhandled Error] {repr(e)}]")
+ import traceback
+
+ # write to error file
+ with open(Path(args.result_dir) / "error.txt", "a") as f:
+ f.write(f"[Config file]: {config_file}\n")
+ f.write(f"[Unhandled Error] {repr(e)}\n")
+ f.write(traceback.format_exc()) # write stack trace to file
+
+ env.close()
+ if len(scores) > 0:
+ logger.info(f"Average score: {sum(scores) / len(scores)}")
+
+
+def prepare(args: argparse.Namespace) -> None:
+ # convert prompt python files to json
+ from agent.prompts import to_json
+
+ to_json.run()
+
+ # prepare result dir
+ result_dir = args.result_dir
+ if not result_dir:
+ result_dir = (
+ f"cache/results_{time.strftime('%Y%m%d%H%M%S', time.localtime())}"
+ )
+ if not Path(result_dir).exists():
+ Path(result_dir).mkdir(parents=True, exist_ok=True)
+ args.result_dir = result_dir
+ logger.info(f"Create result dir: {result_dir}")
+
+ if not (Path(result_dir) / "traces").exists():
+ (Path(result_dir) / "traces").mkdir(parents=True)
+
+ # log the log file
+ with open(os.path.join(result_dir, "log_files.txt"), "a+") as f:
+ f.write(f"{LOG_FILE_NAME}\n")
+
+def get_unfinished(config_files: list[str], result_dir: str) -> list[str]:
+ result_files = glob.glob(f"{result_dir}/traces/*.json")
+ task_ids = [
+ os.path.basename(f).split(".")[0].split("_")[1] for f in result_files
+ ]
+ unfinished_configs = []
+ for config_file in config_files:
+ task_id = os.path.basename(config_file).split(".")[0]
+ if task_id not in task_ids:
+ unfinished_configs.append(config_file)
+ return unfinished_configs
+
+def dump_config(args: argparse.Namespace) -> None:
+ config_file = Path(args.result_dir) / "config.json"
+ if not config_file.exists():
+ with open(config_file, "w") as f:
+ json.dump(vars(args), f, indent=4)
+ logger.info(f"Dump config to {config_file}")
+
+
+if __name__ == "__main__":
+ args = config()
+ args.sleep_after_execution = 2.0
+ prepare(args)
+
+ test_file_list = []
+ st_idx = args.test_start_idx
+ ed_idx = args.test_end_idx
+ for i in range(st_idx, ed_idx):
+ if not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "config_files", f"{i}.json")):
+ continue
+ test_file_list.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "config_files", f"{i}.json"))
+
+ if len(test_file_list) == 0:
+ logger.info("No task left to run")
+ else:
+ print(f"Total {len(test_file_list)} tasks left")
+ args.render = False
+ args.render_screenshot = True
+ args.save_trace_enabled = True
+
+ args.current_viewport_only = True
+ dump_config(args)
+
+ agent = construct_agent(args)
+ test(args, agent, test_file_list)
\n"
+
+ # meta data
+ new_content += f"{meta_data['action_history'][-1]}
\n"
+
+ # action
+ action_str = get_render_action(
+ action,
+ info["observation_metadata"],
+ action_set_tag=self.action_set_tag,
+ )
+ # with yellow background
+ action_str = f"{action_str}
"
+ new_content += f"{action_str}\n"
+
+ # add new content
+ self.render_file.seek(0)
+ html = self.render_file.read()
+ html_body = re.findall(r"(.*?)", html, re.DOTALL)[0]
+ html_body += new_content
+
+ html = HTML_TEMPLATE.format(body=html_body)
+ self.render_file.seek(0)
+ self.render_file.truncate()
+ self.render_file.write(html)
+ self.render_file.flush()
+
+ def close(self) -> None:
+ self.render_file.close()
diff --git a/browser_env/html_tools/__init__.py b/browser_env/html_tools/__init__.py
new file mode 100644
index 0000000..5e76c66
--- /dev/null
+++ b/browser_env/html_tools/__init__.py
@@ -0,0 +1,6 @@
+from .identifier import IdentifierTool
+from .prompt import HtmlPrompt
+from .html_parser import HtmlParser
+
+from .utils import print_html_object
+from .configs import basic_attrs, mind2web_keep_attrs
\ No newline at end of file
diff --git a/browser_env/html_tools/configs/__init__.py b/browser_env/html_tools/configs/__init__.py
new file mode 100644
index 0000000..b900a67
--- /dev/null
+++ b/browser_env/html_tools/configs/__init__.py
@@ -0,0 +1,3 @@
+from .html_prompt import prompts
+from .config import basic_attrs, mind2web_keep_attrs, miniwob_attrs
+from .config import config_meta
\ No newline at end of file
diff --git a/browser_env/html_tools/configs/config.py b/browser_env/html_tools/configs/config.py
new file mode 100644
index 0000000..a979364
--- /dev/null
+++ b/browser_env/html_tools/configs/config.py
@@ -0,0 +1,49 @@
+basic_attrs = [
+ 'title',
+ 'value',
+ 'placeholder',
+ 'selected',
+]
+
+mind2web_keep_attrs = [
+ 'alt',
+ 'aria_description',
+ 'aria_label',
+ 'aria_role',
+ 'input_checked',
+ 'input_value',
+ 'label',
+ 'name',
+ 'option_selected',
+ 'placeholder',
+ 'role',
+ 'text_value',
+ 'title',
+ 'type',
+ 'value',
+]
+
+miniwob_attrs = [
+ 'id',
+ 'type',
+ 'value',
+]
+
+config_meta = """
+======= Configs =======
+Columns:
+ - id: {id_attr}
+ - label: {label_attr}
+Position: {use_position}
+ - window: {window_size}
+ - rect_dict: {rect}
+Keep:
+ - parents: {parent_chain}
+ - attrs: {keep_attrs}
+ - elems: {keep_elem}
+ - obs_elem: {obs_elem}
+Generator:
+ - prompt: {prompt_name}
+ - label: {identifier_name}
+========================
+"""
\ No newline at end of file
diff --git a/browser_env/html_tools/configs/html_prompt.py b/browser_env/html_tools/configs/html_prompt.py
new file mode 100644
index 0000000..904d021
--- /dev/null
+++ b/browser_env/html_tools/configs/html_prompt.py
@@ -0,0 +1,22 @@
+refine_prompt = {
+ 'dom': '<{tag}{label}|{attr}{content}{subtree} >',
+ 'label': '[{label}]',
+ 'attr': '{attr}',
+ 'attr_splitter': '; ',
+ 'subtree_splitter': ' ',
+}
+
+xml_prompt = {
+ 'dom': '<{tag}{label}{attr}>{content}{subtree} ',
+ 'label': ' id="{label}"',
+ 'attr': '{key}="{attr}"',
+ 'attr_splitter': ' ',
+ 'subtree_splitter': ' ',
+}
+
+prompts = {
+ 'refine': refine_prompt,
+ 'xml': xml_prompt,
+ 'new_data': refine_prompt,
+}
+
\ No newline at end of file
diff --git a/browser_env/html_tools/html_parser.py b/browser_env/html_tools/html_parser.py
new file mode 100644
index 0000000..7b173ce
--- /dev/null
+++ b/browser_env/html_tools/html_parser.py
@@ -0,0 +1,426 @@
+from lxml import html
+import time, copy, random
+import json, re, os
+
+from .identifier import IdentifierTool
+from .prompt import HtmlPrompt
+from .configs import config_meta
+from .utils import get_xpath_top_down, rect2tuple
+
+class HtmlParser():
+ def __init__(self, ctx: str, args: dict[str]={}) -> None:
+ stt = time.time()
+ self.dom_tree = self.ctx2tree(ctx)
+ # tool related
+ self.bids2label = {}
+ self.bids2xpath = {}
+ self.used_labels = {}
+
+ # parse args
+ self.parse_args(args)
+ self.init_time = time.time() - stt
+
+ def parse_args(self, args: dict[str]={}) -> None:
+ def attr_check(attr, type_model='str'):
+ if attr is None:
+ return False
+ attr_type = type(attr)
+ if attr_type != type(type_model):
+ return False
+ if attr_type == type('str') and len(attr) == 0:
+ return False
+ return True
+
+ args = {} if args is None else args
+
+ # [Position] use_pos: False -> use full page, otherwise use window_size
+ dataset = args.get('dataset', '')
+ use_position = args.get('use_position', False)
+ window_size = args.get('window_size', None)
+ rect = args.get('rect_dict', None)
+ if use_position:
+ if not attr_check(window_size, ()):
+ raise ValueError('window_size must be set when use_position is True')
+ if not attr_check(rect, {}):
+ raise ValueError('rect_dict must be set when use_position is True')
+
+ if not attr_check(rect, {}):
+ rect = {}
+
+ # [Label] for vimium is temp_clickable_label, otherwise keep all of it
+ label_attr = args.get('label_attr', '')
+ get_new_label = args.get('regenerate_label', False)
+ label_method = args.get('label_generator', None)
+ regen_label = not attr_check(label_method)
+
+ # [id] for mind2web is backend_node_id, for normal website use our method
+ id_attr = args.get('id_attr', '')
+ regen_id = not attr_check(id_attr)
+
+ if regen_id:
+ id_attr = 'temp_id'
+
+ # [attributes]
+ keep_attrs = args.get('attr_list', [])
+ if not attr_check(keep_attrs, []):
+ keep_attrs = []
+
+ # [Tags] for clickable elem, keep: must keep, obs: keep if follow specific rule
+ parent_chain = args.get('parent_chain', False)
+ keep_elem = args.get('keep_elem', [])
+ obs_elem = args.get('obs_elem', [])
+
+ # sanity check
+ self.set_args(use_position, window_size, rect, label_attr, id_attr, keep_attrs, keep_elem, obs_elem, parent_chain, get_new_label, dataset)
+
+ # [Prompt]
+ prompt = args.get('prompt', None)
+ self.prompt = HtmlPrompt(prompt)
+
+ # traverse and get special data
+ if regen_id or regen_label:
+ self.mark_id()
+
+ if get_new_label:
+ self.used_labels = {}
+
+ self.identifier = IdentifierTool(label_method, self.used_labels)
+
+ def set_args(self, use_position: bool=False, window_size: tuple=(), rect_dict: dict[str]={}, label_attr: str='',
+ id_attr: str='', keep_attrs: list[str]=[], keep_elem: list[str]=[], obs_elem: list[str]=[],
+ parent_chain: bool=False, get_new_label: bool=False, dataset: str='') -> None:
+
+ self.use_position = use_position
+ self.window_size = window_size
+ self.rect = rect_dict
+ self.label_attr = label_attr
+ self.id_attr = id_attr
+ self.keep_attrs = keep_attrs
+ self.keep = keep_elem
+ self.obs = obs_elem
+ self.parent_chain = parent_chain
+ self.get_new_label = get_new_label
+ self.dataset = dataset
+
+ def get_config(self):
+ config = {
+ 'id_attr': self.id_attr,
+ 'keep_attrs': self.keep_attrs[:5],
+ 'label_attr': self.label_attr,
+ 'use_position': self.use_position,
+ 'window_size': self.window_size,
+ 'rect': dict(list(self.rect.items())[:3]),
+ 'keep_elem': self.keep[:5],
+ 'obs_elem': self.obs[:5],
+ 'parent_chain': self.parent_chain,
+ 'prompt_name': self.prompt.name,
+ 'identifier_name': self.identifier.name
+ }
+
+ return config, config_meta.format(**config)
+
+ def update_rect_dict(self, rect_dict: dict[str]={}) -> None:
+ self.rect = rect_dict
+
+ @staticmethod
+ def ctx2tree(ctx: str) -> html.HtmlElement:
+ # remove useless tags, eg. style and script
+ ctx = re.sub('', '', ctx)
+ ctx = re.sub('