init
This commit is contained in:
Yao Liu
8
webarena/agent/__init__.py
Normal file
8
webarena/agent/__init__.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from .agent import (
|
||||
Agent,
|
||||
PromptAgent,
|
||||
TeacherForcingAgent,
|
||||
construct_agent,
|
||||
)
|
||||
|
||||
__all__ = ["Agent", "TeacherForcingAgent", "PromptAgent", "construct_agent"]
|
||||
227
webarena/agent/agent.py
Normal file
227
webarena/agent/agent.py
Normal file
@@ -0,0 +1,227 @@
|
||||
import argparse
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import tiktoken
|
||||
from beartype import beartype
|
||||
|
||||
from agent.prompts import *
|
||||
from browser_env import Trajectory
|
||||
from browser_env.actions import (
|
||||
Action,
|
||||
ActionParsingError,
|
||||
create_id_based_action,
|
||||
create_none_action,
|
||||
create_playwright_action,
|
||||
)
|
||||
from browser_env.utils import Observation, StateInfo
|
||||
from llms import (
|
||||
call_llm,
|
||||
generate_from_huggingface_completion,
|
||||
generate_from_openai_chat_completion,
|
||||
generate_from_openai_completion,
|
||||
lm_config,
|
||||
)
|
||||
from llms.tokenizers import Tokenizer
|
||||
|
||||
|
||||
class Agent:
|
||||
"""Base class for the agent"""
|
||||
|
||||
def __init__(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def next_action(
|
||||
self, trajectory: Trajectory, intent: str, meta_data: Any
|
||||
) -> Action:
|
||||
"""Predict the next action given the observation"""
|
||||
raise NotImplementedError
|
||||
|
||||
def check_action(
|
||||
self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any], target_action: str
|
||||
) -> Action:
|
||||
"""Predict the next action given the observation"""
|
||||
raise NotImplementedError
|
||||
|
||||
def reset(
|
||||
self,
|
||||
test_config_file: str,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TeacherForcingAgent(Agent):
|
||||
"""Agent that follows a pre-defined action sequence"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def set_action_set_tag(self, tag: str) -> None:
|
||||
self.action_set_tag = tag
|
||||
|
||||
def set_actions(self, action_seq: str | list[str]) -> None:
|
||||
if isinstance(action_seq, str):
|
||||
action_strs = action_seq.strip().split("\n")
|
||||
else:
|
||||
action_strs = action_seq
|
||||
action_strs = [a.strip() for a in action_strs]
|
||||
|
||||
actions = []
|
||||
for a_str in action_strs:
|
||||
try:
|
||||
if self.action_set_tag == "playwright":
|
||||
cur_action = create_playwright_action(a_str)
|
||||
elif self.action_set_tag == "id_accessibility_tree":
|
||||
cur_action = create_id_based_action(a_str)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown action type {self.action_set_tag}"
|
||||
)
|
||||
except ActionParsingError as e:
|
||||
cur_action = create_none_action()
|
||||
|
||||
cur_action["raw_prediction"] = a_str
|
||||
actions.append(cur_action)
|
||||
|
||||
self.actions: list[Action] = actions
|
||||
|
||||
def next_action(
|
||||
self, trajectory: Trajectory, intent: str, meta_data: Any
|
||||
) -> Action:
|
||||
"""Predict the next action given the observation"""
|
||||
return self.actions.pop(0)
|
||||
|
||||
def reset(
|
||||
self,
|
||||
test_config_file: str,
|
||||
) -> None:
|
||||
with open(test_config_file) as f:
|
||||
ref_actions = json.load(f)["reference_action_sequence"]
|
||||
tag = ref_actions["action_set_tag"]
|
||||
action_seq = ref_actions["action_sequence"]
|
||||
self.set_action_set_tag(tag)
|
||||
self.set_actions(action_seq)
|
||||
|
||||
|
||||
class PromptAgent(Agent):
|
||||
"""prompt-based agent that emits action given the history"""
|
||||
|
||||
@beartype
|
||||
def __init__(
|
||||
self,
|
||||
action_set_tag: str,
|
||||
lm_config: lm_config.LMConfig,
|
||||
prompt_constructor: PromptConstructor,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.lm_config = lm_config
|
||||
self.prompt_constructor = prompt_constructor
|
||||
self.action_set_tag = action_set_tag
|
||||
|
||||
def set_action_set_tag(self, tag: str) -> None:
|
||||
self.action_set_tag = tag
|
||||
|
||||
@beartype
|
||||
def next_action(
|
||||
self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any]
|
||||
) -> Action:
|
||||
prompt = self.prompt_constructor.construct(
|
||||
trajectory, intent, meta_data
|
||||
)
|
||||
lm_config = self.lm_config
|
||||
n = 0
|
||||
while True:
|
||||
response = call_llm(lm_config, prompt)
|
||||
force_prefix = self.prompt_constructor.instruction[
|
||||
"meta_data"
|
||||
].get("force_prefix", "")
|
||||
response = f"{force_prefix}{response}"
|
||||
n += 1
|
||||
try:
|
||||
parsed_response = self.prompt_constructor.extract_action(
|
||||
response
|
||||
)
|
||||
if self.action_set_tag in ["id_html_tree", "id_html_nasc_tree", "id_accessibility_tree"]:
|
||||
action = create_id_based_action(parsed_response)
|
||||
elif self.action_set_tag == "playwright":
|
||||
action = create_playwright_action(parsed_response)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown action type {self.action_set_tag}"
|
||||
)
|
||||
action["raw_prediction"] = response
|
||||
break
|
||||
except ActionParsingError as e:
|
||||
if n >= lm_config.gen_config["max_retry"]:
|
||||
action = create_none_action()
|
||||
action["raw_prediction"] = response
|
||||
break
|
||||
|
||||
return action
|
||||
|
||||
def check_action(
|
||||
self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any], target_action: str
|
||||
) -> Action:
|
||||
prompt = self.prompt_constructor.construct(
|
||||
trajectory, intent, meta_data
|
||||
)
|
||||
lm_config = self.lm_config
|
||||
n = 0
|
||||
|
||||
# agent will retry if the action is not parsed correctly
|
||||
while True:
|
||||
response = target_action
|
||||
force_prefix = self.prompt_constructor.instruction[
|
||||
"meta_data"
|
||||
].get("force_prefix", "")
|
||||
response = f"{force_prefix}{response}"
|
||||
n += 1
|
||||
try:
|
||||
parsed_response = self.prompt_constructor.extract_action(
|
||||
response
|
||||
)
|
||||
if self.action_set_tag in ["id_accessibility_tree", "id_html_tree", "id_html_nasc_tree"]:
|
||||
action = create_id_based_action(parsed_response)
|
||||
elif self.action_set_tag == "playwright":
|
||||
action = create_playwright_action(parsed_response)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown action type {self.action_set_tag}"
|
||||
)
|
||||
action["raw_prediction"] = response
|
||||
break
|
||||
except ActionParsingError as e:
|
||||
if n >= lm_config.gen_config["max_retry"]:
|
||||
action = create_none_action()
|
||||
action["raw_prediction"] = response
|
||||
break
|
||||
|
||||
return prompt, action
|
||||
|
||||
def reset(self, test_config_file: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def construct_agent(args: argparse.Namespace) -> Agent:
|
||||
llm_config = lm_config.construct_llm_config(args)
|
||||
|
||||
agent: Agent
|
||||
if args.agent_type == "teacher_forcing":
|
||||
agent = TeacherForcingAgent()
|
||||
elif args.agent_type == "prompt":
|
||||
with open(args.instruction_path) as f:
|
||||
constructor_type = json.load(f)["meta_data"]["prompt_constructor"]
|
||||
tokenizer = Tokenizer(args.provider, args.model)
|
||||
prompt_constructor = eval(constructor_type)(
|
||||
args.instruction_path, lm_config=llm_config, tokenizer=tokenizer
|
||||
)
|
||||
agent = PromptAgent(
|
||||
action_set_tag=args.action_set_tag,
|
||||
lm_config=llm_config,
|
||||
prompt_constructor=prompt_constructor,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"agent type {args.agent_type} not implemented"
|
||||
)
|
||||
return agent
|
||||
1
webarena/agent/prompts/__init__.py
Normal file
1
webarena/agent/prompts/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .prompt_constructor import *
|
||||
21
webarena/agent/prompts/jsons/new_action_prompt.json
Normal file
21
webarena/agent/prompts/jsons/new_action_prompt.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"intro": "",
|
||||
"examples": [],
|
||||
"template": "<html> {html} </html>\n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description.\nYour goal is to complete the task. You can use the provided functions below to interact with the current webpage.\n\n#Provided functions:\ndef click(element_id: str) -> None:\n \"\"\"\n Click on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef hover(element_id: str) -> None:\n \"\"\"\n Hover on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef select(element_id: str, option: str) -> None:\n \"\"\"\n Select an option from a dropdown.\n\n Args:\n element_id: The id of the element.\n option: Value of the option to select.\n \"\"\"\n\ndef type_string(element_id: str, content: str, press_enter: bool) -> None:\n \"\"\"\n Type a string into the element with the specified id.\n\n Args:\n element_id: The id of the element.\n content: The string to type.\n press_enter: Whether to press enter after typing the string.\n \"\"\"\n\ndef scroll_page(direction: Literal['up', 'down']) -> None:\n \"\"\"\n Scroll down/up one page.\n\n Args:\n direction: The direction to scroll.\n \"\"\"\n\ndef go(direction: Literal['forward', 'backward']) -> None:\n \"\"\"\n Go forward/backward\n\n Args:\n direction: The direction to go to.\n \"\"\"\n\ndef jump_to(url: str, new_tab: bool) -> None:\n \"\"\"\n Jump to the specified url.\n\n Args:\n url: The url to jump to.\n new_tab: Whether to open the url in a new tab.\n \"\"\"\n\ndef switch_tab(tab_index: int) -> None:\n \"\"\"\n Switch to the specified tab.\n\n Args:\n tab_index: The index of the tab to switch to.\n \"\"\"\n\ndef user_input(message: str) -> str:\n \"\"\"\n Wait for user input.\n\n Args:\n message: The message to display to the user.\n\n Returns: The user input.\n \"\"\"\n\ndef finish(answer: Optional[str]) -> None:\n \"\"\"\n Finish the task (optionally with an answer).\n\n Args:\n answer: The answer to the task.\n \"\"\"\n\n#Previous commands: {previous_action}\n\n#Window tabs: {tabs}\n\n#Current viewport (pages): {position}\n\n#Task: {objective}\n\nYou should output one command to interact to the currrent webpage.\nYou should add a brief comment to your command to explain your reasoning and thinking process.\n",
|
||||
"finale": "",
|
||||
"meta_data": {
|
||||
"observation": "html",
|
||||
"action_type": "id_html_nasc_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"html",
|
||||
"objective",
|
||||
"position",
|
||||
"previous_action",
|
||||
"tabs"
|
||||
],
|
||||
"prompt_constructor": "NewASPromptConstructor",
|
||||
"answer_phrase": "",
|
||||
"action_splitter": "#"
|
||||
}
|
||||
}
|
||||
27
webarena/agent/prompts/jsons/p_cot_id_actree_2s.json
Normal file
27
webarena/agent/prompts/jsons/p_cot_id_actree_2s.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
|
||||
"examples": [
|
||||
[
|
||||
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
|
||||
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
|
||||
],
|
||||
[
|
||||
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
|
||||
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
|
||||
]
|
||||
],
|
||||
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"objective",
|
||||
"observation",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
}
|
||||
}
|
||||
27
webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json
Normal file
27
webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
|
||||
"examples": [
|
||||
[
|
||||
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
|
||||
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
|
||||
],
|
||||
[
|
||||
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
|
||||
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
|
||||
]
|
||||
],
|
||||
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"objective",
|
||||
"observation",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
}
|
||||
}
|
||||
26
webarena/agent/prompts/jsons/p_direct_id_actree_2s.json
Normal file
26
webarena/agent/prompts/jsons/p_direct_id_actree_2s.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
|
||||
"examples": [
|
||||
[
|
||||
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
|
||||
"```stop [$279.49]```"
|
||||
],
|
||||
[
|
||||
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
|
||||
"```type [164] [restaurants near CMU] [1]```"
|
||||
]
|
||||
],
|
||||
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"objective",
|
||||
"observation",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "DirectPromptConstructor",
|
||||
"action_splitter": "```"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective.",
|
||||
"examples": [
|
||||
[
|
||||
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
|
||||
"```stop [$279.49]```"
|
||||
],
|
||||
[
|
||||
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
|
||||
"```type [164] [restaurants near CMU] [1]```"
|
||||
]
|
||||
],
|
||||
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"objective",
|
||||
"observation",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\n\nYou can only issue one action at a time",
|
||||
"examples": [
|
||||
[
|
||||
"Observation:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t[1749] StaticText '$279.49'\n\t[1757] button 'Add to Cart'\n\t[1760] button 'Add to Wish List'\n\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nObjective: What is the price of HP Inkjet Fax Machine\nPrevious action: None",
|
||||
"```stop [$279.49]```"
|
||||
],
|
||||
[
|
||||
"Observation:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nObjective: Show me the restaurants near CMU\nPrevious action: None",
|
||||
"```type [164] [restaurants near CMU] [1]```"
|
||||
],
|
||||
[
|
||||
"Observation:\n[2036] button 'Sort by: New' hasPopup: menu expanded: False\n\t[587] link 'US Marine\u2019s adoption of Afghan war orphan voided'\n\t\t[989] time 'March 30, 2023 at 15:03:48 AM UTC'\n\t[602] link 'York student uses AI chatbot to get parking fine revoked'\n\t\t[1025] time 'March 15, 2023 at 7:48:34 AM UTC'\n\t[617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'\n\t\t[1025] time 'March 2, 2023 at 3:46:01 AM UTC'\nURL: http://reddit.com/f/news/new\nObjective: Open the most recent post that was published prior to March 1st.\nPrevious action: None",
|
||||
"```scroll [down]```"
|
||||
]
|
||||
],
|
||||
"template": "Observation:\n{observation}\nURL: {url}\nObjective: {objective}\nPrevious action: {previous_action}",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"objective",
|
||||
"observation",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "DirectPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```",
|
||||
"force_prefix": "```"
|
||||
}
|
||||
}
|
||||
20
webarena/agent/prompts/jsons/test_prompt.json
Normal file
20
webarena/agent/prompts/jsons/test_prompt.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"intro": "",
|
||||
"examples": [],
|
||||
"template": "<html> {html} </html>\n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description. \nYour goal is to complete the task. You can perform the specified operations below to interact with the webpage.\n\n#Valid operations: - #Click# id: Click on the element with the specified id\n- #Scroll_up#: Scroll up 1 page.\n- #Scroll_down#: Scroll down 1 page.\n- #Go_backward#: Go back to the previous page.\n- #Go_forward#: Go forward to the next page.\n- #Hover# id: Hover over the element with the specified id.\n- #Type# id \"text\": Type in the text at the element with the specified id.\n- #Select# id \"option\": Select the option at the element with the specified id.\n- #Record# \"content\": Mark content that is useful in answering the question.\n- #Answer# \"text\": output the text as the answer to the user.\n- #Exit#: Complete the task and exit the program.\n\n#Current viewport position: {position}\n\n#Previous Operation: {previous_action}\n\n#Task: {objective}\n",
|
||||
"finale": "\nYour output SHOULD be in the following format:\n#Operation: {Next operation to perform}\n",
|
||||
"meta_data": {
|
||||
"observation": "html",
|
||||
"action_type": "id_html_tree",
|
||||
"keywords": [
|
||||
"url",
|
||||
"html",
|
||||
"objective",
|
||||
"position",
|
||||
"previous_action"
|
||||
],
|
||||
"prompt_constructor": "MyPromptConstructor",
|
||||
"answer_phrase": "",
|
||||
"action_splitter": "#"
|
||||
}
|
||||
}
|
||||
683
webarena/agent/prompts/prompt_constructor.py
Normal file
683
webarena/agent/prompts/prompt_constructor.py
Normal file
@@ -0,0 +1,683 @@
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, TypedDict
|
||||
|
||||
from browser_env import Action, ActionParsingError, Trajectory
|
||||
from browser_env.env_config import URL_MAPPINGS
|
||||
from browser_env.utils import StateInfo
|
||||
from llms import lm_config
|
||||
from llms.tokenizers import Tokenizer
|
||||
from llms.utils import APIInput
|
||||
|
||||
|
||||
class Instruction(TypedDict):
|
||||
"""Instruction for constructing prompt"""
|
||||
|
||||
intro: str
|
||||
examples: list[tuple[str, str]]
|
||||
template: str
|
||||
meta_data: dict[str, Any]
|
||||
|
||||
|
||||
class PromptConstructor(object):
|
||||
def __init__(
|
||||
self,
|
||||
instruction_path: str | Path,
|
||||
lm_config: lm_config.LMConfig,
|
||||
tokenizer: Tokenizer,
|
||||
):
|
||||
self.instruction_path = Path(instruction_path)
|
||||
self.obs_modality = "text"
|
||||
self.lm_config = lm_config
|
||||
instruction = json.load(open(self.instruction_path))
|
||||
instruction["examples"] = [tuple(e) for e in instruction["examples"]]
|
||||
self.instruction: Instruction = instruction
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def get_lm_api_input(
|
||||
self, intro: str, examples: list[tuple[str, str]], current: str
|
||||
) -> APIInput:
|
||||
|
||||
"""Return the require format for an API"""
|
||||
message: list[dict[str, str]] | str
|
||||
if "openai" in self.lm_config.provider:
|
||||
if self.lm_config.mode == "chat":
|
||||
message = [{"role": "system", "content": intro}]
|
||||
for (x, y) in examples:
|
||||
message.append(
|
||||
{
|
||||
"role": "system",
|
||||
"name": "example_user",
|
||||
"content": x,
|
||||
}
|
||||
)
|
||||
message.append(
|
||||
{
|
||||
"role": "system",
|
||||
"name": "example_assistant",
|
||||
"content": y,
|
||||
}
|
||||
)
|
||||
message.append({"role": "user", "content": current})
|
||||
return message
|
||||
elif self.lm_config.mode == "completion":
|
||||
message = f"{intro}\n\n"
|
||||
message += "Here are a few examples:\n"
|
||||
for example in examples:
|
||||
message += f"Observation\n:{example[0]}\n\n"
|
||||
message += f"Action: {example[1]}\n\n"
|
||||
message += "Now make prediction given the observation\n\n"
|
||||
message += f"Observation\n:{current}\n\n"
|
||||
message += "Action:"
|
||||
return message
|
||||
else:
|
||||
raise ValueError(
|
||||
f"OpenAI models do not support mode {self.lm_config.mode}"
|
||||
)
|
||||
elif "huggingface" in self.lm_config.provider:
|
||||
# https://huggingface.co/blog/llama2#how-to-prompt-llama-2
|
||||
# https://github.com/facebookresearch/llama/blob/main/llama/generation.py#L320
|
||||
if "Llama-2" in self.lm_config.model:
|
||||
if self.lm_config.mode == "chat":
|
||||
B_INST, E_INST = "[INST]", "[/INST]"
|
||||
B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
|
||||
BOS, EOS = "<s>", "</s>"
|
||||
# adding the system message to be the starting of the first example
|
||||
examples = [
|
||||
(
|
||||
B_SYS + intro + E_SYS + examples[0][0],
|
||||
examples[0][1],
|
||||
)
|
||||
] + examples[1:]
|
||||
message = "".join(
|
||||
[
|
||||
f"{BOS}{B_INST} {x.strip()} {E_INST} {y.strip()} {EOS}"
|
||||
for (x, y) in examples
|
||||
]
|
||||
)
|
||||
# add the current observation
|
||||
message += f"{BOS}{B_INST} {current.strip()} {E_INST} {self.instruction['meta_data'].get('force_prefix', '')}"
|
||||
|
||||
return message
|
||||
else:
|
||||
raise ValueError("Only chat mode is supported for Llama-2")
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Huggingface models do not support model_tag {self.lm_config.gen_config['model_tag']}"
|
||||
)
|
||||
elif "ours" in self.lm_config.provider:
|
||||
message = f"{intro}\n\n"
|
||||
message += "Now make prediction given the observation\n\n"
|
||||
message += f"Observation\n:{current}\n\n"
|
||||
message += "Action:"
|
||||
return message
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Provider {self.lm_config.provider} not implemented"
|
||||
)
|
||||
|
||||
def construct(
|
||||
self,
|
||||
trajectory: Trajectory,
|
||||
intent: str,
|
||||
meta_data: dict[str, Any] = {},
|
||||
) -> APIInput:
|
||||
raise NotImplementedError
|
||||
|
||||
def map_url_to_real(self, url: str) -> str:
|
||||
"""Map the urls to their real world counterparts"""
|
||||
for i, j in URL_MAPPINGS.items():
|
||||
if i in url:
|
||||
url = url.replace(i, j)
|
||||
return url
|
||||
|
||||
def map_url_to_local(self, url: str) -> str:
|
||||
"""Map the urls to their local counterparts"""
|
||||
for i, j in URL_MAPPINGS.items():
|
||||
if j in url:
|
||||
url = url.replace(j, i)
|
||||
# https
|
||||
if j.replace("http", "https") in url:
|
||||
url = url.replace(j.replace("http", "https"), i)
|
||||
return url
|
||||
|
||||
def _extract_action(self, response: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def extract_action(self, response: str) -> str:
|
||||
response = self._extract_action(response)
|
||||
response = self.map_url_to_local(response)
|
||||
return response
|
||||
|
||||
|
||||
class DirectPromptConstructor(PromptConstructor):
|
||||
"""The agent will direct predict the action"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instruction_path: str | Path,
|
||||
lm_config: lm_config.LMConfig,
|
||||
tokenizer: Tokenizer,
|
||||
):
|
||||
super().__init__(instruction_path, lm_config, tokenizer)
|
||||
|
||||
def construct(
|
||||
self,
|
||||
trajectory: Trajectory,
|
||||
intent: str,
|
||||
meta_data: dict[str, Any] = {},
|
||||
) -> APIInput:
|
||||
"""Construct prompt given the trajectory"""
|
||||
intro = self.instruction["intro"]
|
||||
examples = self.instruction["examples"]
|
||||
template = self.instruction["template"]
|
||||
keywords = self.instruction["meta_data"]["keywords"]
|
||||
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
|
||||
|
||||
obs = state_info["observation"][self.obs_modality]
|
||||
max_obs_length = self.lm_config.gen_config["max_obs_length"]
|
||||
if max_obs_length:
|
||||
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
|
||||
|
||||
page = state_info["info"]["page"]
|
||||
url = page.url
|
||||
previous_action_str = meta_data["action_history"][-1]
|
||||
|
||||
# input x
|
||||
current = template.format(
|
||||
objective=intent,
|
||||
url=self.map_url_to_real(url),
|
||||
observation=obs,
|
||||
previous_action=previous_action_str,
|
||||
)
|
||||
|
||||
# make sure all keywords are replaced
|
||||
assert all([f"{{k}}" not in current for k in keywords])
|
||||
prompt = self.get_lm_api_input(intro, examples, current)
|
||||
return prompt
|
||||
|
||||
def _extract_action(self, response: str) -> str:
|
||||
action_splitter = self.instruction["meta_data"]["action_splitter"]
|
||||
pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
|
||||
match = re.search(pattern, response)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
raise ActionParsingError(
|
||||
f"Cannot parse action from response {response}"
|
||||
)
|
||||
|
||||
|
||||
class CoTPromptConstructor(PromptConstructor):
|
||||
"""The agent will perform step-by-step reasoning before the answer"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instruction_path: str | Path,
|
||||
lm_config: lm_config.LMConfig,
|
||||
tokenizer: Tokenizer,
|
||||
):
|
||||
super().__init__(instruction_path, lm_config, tokenizer)
|
||||
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
|
||||
|
||||
def construct(
|
||||
self,
|
||||
trajectory: Trajectory,
|
||||
intent: str,
|
||||
meta_data: dict[str, Any] = {},
|
||||
) -> APIInput:
|
||||
intro = self.instruction["intro"]
|
||||
examples = self.instruction["examples"]
|
||||
template = self.instruction["template"]
|
||||
keywords = self.instruction["meta_data"]["keywords"]
|
||||
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
|
||||
|
||||
obs = state_info["observation"][self.obs_modality]
|
||||
max_obs_length = self.lm_config.gen_config["max_obs_length"]
|
||||
if max_obs_length:
|
||||
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
|
||||
|
||||
page = state_info["info"]["page"]
|
||||
url = page.url
|
||||
previous_action_str = meta_data["action_history"][-1]
|
||||
current = template.format(
|
||||
objective=intent,
|
||||
url=self.map_url_to_real(url),
|
||||
observation=obs,
|
||||
previous_action=previous_action_str,
|
||||
)
|
||||
|
||||
assert all([f"{{k}}" not in current for k in keywords])
|
||||
|
||||
prompt = self.get_lm_api_input(intro, examples, current)
|
||||
return prompt
|
||||
|
||||
def _extract_action(self, response: str) -> str:
|
||||
# find the first occurence of action
|
||||
action_splitter = self.instruction["meta_data"]["action_splitter"]
|
||||
pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
|
||||
match = re.search(pattern, response)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
else:
|
||||
raise ActionParsingError(
|
||||
f'Cannot find the answer phrase "{self.answer_phrase}" in "{response}"'
|
||||
)
|
||||
|
||||
class MyPromptConstructor(PromptConstructor):
|
||||
"""The agent will perform step-by-step reasoning before the answer"""
|
||||
operation = [
|
||||
r"#?(Click)#?\s*([0-9]+)",
|
||||
r"#?(Type)#?\s*([0-9]+)\s+[\'\"]{0,1}([\s\S]+)[\'\"]{0,1}",
|
||||
r"#?(Select)#?\s*([0-9]+)\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
|
||||
r"#?(Scroll_up)#?",
|
||||
r"#?(Scroll_down)#?",
|
||||
r"#?(Goto)#?\s*(https?:\/\/[-a-z0-9]+(?:\.[-a-z0-9]+)*\.(?:com|cn|edu|uk)(?:\/[-a-z0-9_:@&?=+,.!/~*'%$]*)?)",
|
||||
r"#?(Go_backward)#?",
|
||||
r"#?(Go_forward)#?",
|
||||
r"#?(Hover)#?\s*([0-9]+)",
|
||||
r"#?(Answer)#?\s+(.+)",
|
||||
r"#?(Login)#?",
|
||||
r"#?(Verify)#?",
|
||||
r"#?(Exit)#?",
|
||||
r"#?(Record)#?\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
|
||||
]
|
||||
|
||||
translate = [
|
||||
"click",
|
||||
"type",
|
||||
"select",
|
||||
"scroll [up]",
|
||||
"scroll [down]",
|
||||
"goto",
|
||||
"go_back",
|
||||
"go_forward",
|
||||
"hover",
|
||||
"stop",
|
||||
"stop",
|
||||
"stop",
|
||||
"stop",
|
||||
"record",
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instruction_path: str | Path,
|
||||
lm_config: lm_config.LMConfig,
|
||||
tokenizer: Tokenizer,
|
||||
):
|
||||
super().__init__(instruction_path, lm_config, tokenizer)
|
||||
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
|
||||
self.state = {}
|
||||
|
||||
def construct(
|
||||
self,
|
||||
trajectory: Trajectory,
|
||||
intent: str,
|
||||
meta_data: dict[str, Any] = {},
|
||||
) -> APIInput:
|
||||
intro = self.instruction["intro"]
|
||||
examples = self.instruction["examples"]
|
||||
template = self.instruction["template"]
|
||||
keywords = self.instruction["meta_data"]["keywords"]
|
||||
finale = self.instruction["finale"]
|
||||
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
|
||||
|
||||
obs = state_info["observation"][self.obs_modality]
|
||||
max_obs_length = self.lm_config.gen_config["max_obs_length"]
|
||||
if max_obs_length:
|
||||
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
|
||||
|
||||
info = state_info["info"]
|
||||
obs_metadata = info["observation_metadata"]["text"]
|
||||
nodes = obs_metadata["obs_nodes_info"]
|
||||
position_info = obs_metadata["position_info"]
|
||||
html_parser = obs_metadata["html_parser"]
|
||||
self.nodes = nodes
|
||||
|
||||
page = info["page"]
|
||||
url = self.map_url_to_real(page.url)
|
||||
position_bar = self._get_position_bar(position_info)
|
||||
|
||||
history = [f"{ix}. {his}" for his in meta_data["action_history"]]
|
||||
if len(history) == 1:
|
||||
previous_action_str = "None"
|
||||
else:
|
||||
previous_action_str = '\n'.join(history[1:])
|
||||
|
||||
self.state.update({
|
||||
"url": url,
|
||||
"html": obs,
|
||||
"html_parser": html_parser,
|
||||
"segment": "None",
|
||||
"operation": "None",
|
||||
})
|
||||
|
||||
current = template.format(
|
||||
objective=intent,
|
||||
url=url,
|
||||
html=obs,
|
||||
position=position_bar,
|
||||
previous_action=previous_action_str,
|
||||
)
|
||||
|
||||
assert all([f"{{k}}" not in current for k in keywords])
|
||||
|
||||
# prompt = self.get_lm_api_input(intro, examples, current)
|
||||
prompt = current + finale
|
||||
|
||||
return prompt
|
||||
|
||||
def _extract_action(self, response: str) -> str:
|
||||
# find the first occurence of action
|
||||
self.state["intention"] = self._extract_intention(response)
|
||||
|
||||
for regex, act in zip(self.operation, self.translate):
|
||||
match = re.search(regex, response)
|
||||
|
||||
if match:
|
||||
m = match.groups()
|
||||
if isinstance(m, tuple):
|
||||
exact_act = m[0]
|
||||
param = m[1:]
|
||||
else:
|
||||
exact_act = m
|
||||
param = []
|
||||
|
||||
param = list(param)
|
||||
if act in ['click', 'hover', 'type', 'select']:
|
||||
if len(param) == 0:
|
||||
continue
|
||||
|
||||
for node_id, node in self.nodes.items():
|
||||
if node['label'] == param[0]:
|
||||
label = param[0]
|
||||
hp = self.state["html_parser"]
|
||||
bid = hp.id_label_converter(label)
|
||||
segment = hp.get_segment(bid)
|
||||
|
||||
print('[Label]', label, bid, segment)
|
||||
self.state["segment"] = segment
|
||||
#self._extract_segment(self.state["html"], label)
|
||||
if act not in ['select']:
|
||||
param[0] = node_id
|
||||
break
|
||||
|
||||
|
||||
if act in ['stop', 'select', 'record']:
|
||||
if len(param) > 0:
|
||||
param[-1] = param[-1].strip("\'\"")
|
||||
|
||||
if act in ['type']:
|
||||
print('In prompt constructer', param[-1])
|
||||
if len(param) > 0:
|
||||
param[-1] = param[-1].strip("\'\"")
|
||||
print(param[-1])
|
||||
if param[-1].endswith('\n'):
|
||||
param[-1] = param[-1][:-1]
|
||||
param.append('1')
|
||||
else:
|
||||
param.append('0')
|
||||
|
||||
command = act
|
||||
for p in param:
|
||||
command += f" [{p}]"
|
||||
|
||||
print(command)
|
||||
return command
|
||||
|
||||
raise ActionParsingError(
|
||||
f'Cannot find the answer phrase in "{response}"'
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_position_bar(data):
|
||||
position = data.get("position", 0.0)
|
||||
page_height = data.get("page_height", 1.0)
|
||||
left_bar = '-' * int(position)
|
||||
right_bar = '-' * int(max(1, page_height - position))
|
||||
return f'[0{left_bar}|{round(position, 1)}{right_bar}{round(page_height, 1)}]'
|
||||
|
||||
@staticmethod
|
||||
def _extract_intention(response, lang='en'):
|
||||
if lang == 'en':
|
||||
matches = re.findall(r"#Thinking Process:\s*(.+)\s*#Operation:", response)
|
||||
print('[Try to match]', matches)
|
||||
else:
|
||||
matches = re.findall(r"#思考过程: (.+)", response)
|
||||
|
||||
if matches:
|
||||
return matches[-1]
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_segment(html: str, tag: str):
|
||||
tag = f'[{tag}]'
|
||||
has_content = False
|
||||
|
||||
def _left(html, start):
|
||||
nonlocal has_content
|
||||
left_cnt, right_cnt = 0, 0
|
||||
for i in range(start, -1, -1):
|
||||
if html[i] == '<':
|
||||
left_cnt += 1
|
||||
elif html[i] == '>':
|
||||
if html[i - 2] != '|' and html[i - 2] != '>':
|
||||
has_content = True
|
||||
right_cnt += 1
|
||||
elif html[i] == '|':
|
||||
if html[i + 2] != '<' and html[i + 2] != '>':
|
||||
has_content = True
|
||||
if left_cnt == right_cnt + 1:
|
||||
return i
|
||||
return -1
|
||||
|
||||
def _right(html, start):
|
||||
nonlocal has_content
|
||||
left_cnt, right_cnt = 0, 0
|
||||
for i in range(start, len(html), 1):
|
||||
if html[i] == '<':
|
||||
left_cnt += 1
|
||||
elif html[i] == '>':
|
||||
if html[i - 2] != '|' and html[i - 2] != '>':
|
||||
has_content = True
|
||||
right_cnt += 1
|
||||
elif html[i] == '|':
|
||||
if html[i + 2] != '<' and html[i + 2] != '>':
|
||||
has_content = True
|
||||
if left_cnt + 1 == right_cnt:
|
||||
return i + 1
|
||||
return -1
|
||||
|
||||
tag_start = html.find(tag)
|
||||
|
||||
if tag_start == -1:
|
||||
return None
|
||||
|
||||
left_bound, right_bound = _left(html, tag_start), _right(html, tag_start)
|
||||
while True:
|
||||
if left_bound == -1 or right_bound == -1:
|
||||
return None
|
||||
|
||||
if has_content:
|
||||
break
|
||||
|
||||
else:
|
||||
lb, rb = _left(html, left_bound - 1), _right(html, right_bound + 1)
|
||||
if lb == -1 or rb == -1:
|
||||
break
|
||||
if rb - lb > 150:
|
||||
break
|
||||
else:
|
||||
left_bound, right_bound = lb, rb
|
||||
|
||||
segment = html[left_bound:right_bound]
|
||||
|
||||
if len(segment) > 150:
|
||||
return segment[:150] + '...>'
|
||||
|
||||
return segment
|
||||
|
||||
class NewASPromptConstructor(PromptConstructor):
|
||||
"""The agent will perform step-by-step reasoning before the answer"""
|
||||
operation = [
|
||||
r"(click)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
|
||||
r"(type_string)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*,\s*(True|False)\s*\)",
|
||||
r"(select)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*\)",
|
||||
r"(scroll_page)\(\s*[\'\"]up[\'\"]\s*\)",
|
||||
r"(scroll_page)\(\s*[\'\"]down[\'\"]\s*\)",
|
||||
r"(jump_to)\(\s*[\'\"](.+)[\'\"]\s*,\s*(True|False)\s*\)",
|
||||
r"(go)\(\s*[\'\"]backward[\'\"]\s*\)",
|
||||
r"(go)\(\s*[\'\"]forward[\'\"]\s*\)",
|
||||
r"(hover)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
|
||||
r"(finish)\(\s*\)",
|
||||
r"(finish)\(\s*(.+)\s*\)",
|
||||
r"(record)\(\s*[\'\"](.+)[\'\"]\s*\)",
|
||||
r"(switch_tab)\([\d]+\)"
|
||||
]
|
||||
|
||||
translate = [
|
||||
"click",
|
||||
"type",
|
||||
"select",
|
||||
"scroll [up]",
|
||||
"scroll [down]",
|
||||
"goto",
|
||||
"go_back",
|
||||
"go_forward",
|
||||
"hover",
|
||||
"stop",
|
||||
"stop",
|
||||
"record",
|
||||
"page_focus",
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instruction_path: str | Path,
|
||||
lm_config: lm_config.LMConfig,
|
||||
tokenizer: Tokenizer,
|
||||
):
|
||||
super().__init__(instruction_path, lm_config, tokenizer)
|
||||
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
|
||||
self.state = {}
|
||||
|
||||
def construct(
|
||||
self,
|
||||
trajectory: Trajectory,
|
||||
intent: str,
|
||||
meta_data: dict[str, Any] = {},
|
||||
) -> APIInput:
|
||||
intro = self.instruction["intro"]
|
||||
examples = self.instruction["examples"]
|
||||
template = self.instruction["template"]
|
||||
keywords = self.instruction["meta_data"]["keywords"]
|
||||
finale = self.instruction["finale"]
|
||||
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
|
||||
|
||||
obs = state_info["observation"][self.obs_modality]
|
||||
max_obs_length = self.lm_config.gen_config["max_obs_length"]
|
||||
if max_obs_length:
|
||||
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
|
||||
|
||||
info = state_info["info"]
|
||||
obs_metadata = info["observation_metadata"]["text"]
|
||||
nodes = obs_metadata["obs_nodes_info"]
|
||||
position_info = obs_metadata["position_info"]
|
||||
html_parser = obs_metadata["html_parser"]
|
||||
tabs_str = obs_metadata["tab_title"]
|
||||
self.nodes = nodes
|
||||
|
||||
page = info["page"]
|
||||
url = self.map_url_to_real(page.url)
|
||||
position_bar = self._get_position_bar(position_info)
|
||||
|
||||
history = meta_data["action_history"]
|
||||
if len(history) == 1:
|
||||
previous_action_str = "None"
|
||||
else:
|
||||
previous_action_str = '\n'.join(history[1:])
|
||||
|
||||
self.state.update({
|
||||
"url": url,
|
||||
"html": obs,
|
||||
"html_parser": html_parser,
|
||||
"segment": "None",
|
||||
"operation": "None",
|
||||
})
|
||||
|
||||
current = template.format(
|
||||
objective=intent,
|
||||
url=url,
|
||||
html=obs,
|
||||
position=position_bar,
|
||||
previous_action=previous_action_str,
|
||||
tabs=tabs_str,
|
||||
)
|
||||
|
||||
assert all([f"{{k}}" not in current for k in keywords])
|
||||
|
||||
# prompt = self.get_lm_api_input(intro, examples, current)
|
||||
prompt = current + finale
|
||||
|
||||
return prompt
|
||||
|
||||
def _extract_action(self, response: str) -> str:
|
||||
# find the first occurence of action
|
||||
# self.state["intention"] = self._extract_intention(response)
|
||||
|
||||
for regex, act in zip(self.operation, self.translate):
|
||||
match = re.search(regex, response)
|
||||
if match:
|
||||
m = match.groups()
|
||||
if isinstance(m, tuple):
|
||||
exact_act = m[0]
|
||||
param = m[1:]
|
||||
else:
|
||||
exact_act = m
|
||||
param = []
|
||||
|
||||
print(exact_act, param)
|
||||
param = list(param)
|
||||
if act in ['click', 'hover', 'type', 'select']:
|
||||
if len(param) == 0:
|
||||
continue
|
||||
|
||||
for node_id, node in self.nodes.items():
|
||||
if node['label'] == param[0]:
|
||||
label = param[0]
|
||||
hp = self.state["html_parser"]
|
||||
bid = hp.id_label_converter(label)
|
||||
segment = hp.get_segment(bid)
|
||||
|
||||
print('[Label]', label, bid, segment)
|
||||
self.state["segment"] = segment
|
||||
#self._extract_segment(self.state["html"], label)
|
||||
if act not in ['select']:
|
||||
param[0] = node_id
|
||||
break
|
||||
|
||||
if len(param) > 0:
|
||||
if act in ['stop', 'select', 'record']:
|
||||
param[-1] = param[-1].strip("\'\"")
|
||||
if act in ['type', 'goto']:
|
||||
param[-1] = '1' if param[-1] == 'True' else '0'
|
||||
|
||||
command = act
|
||||
for p in param:
|
||||
command += f" [{p}]"
|
||||
|
||||
print(command)
|
||||
return command
|
||||
|
||||
raise ActionParsingError(
|
||||
f'Cannot find the answer phrase in "{response}"'
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_position_bar(data):
|
||||
position = data.get("position", 0.0)
|
||||
page_height = data.get("page_height", 1.0)
|
||||
return f"{round(position, 1)} / {round(page_height, 1)}"
|
||||
118
webarena/agent/prompts/raw/new_action_prompt.py
Normal file
118
webarena/agent/prompts/raw/new_action_prompt.py
Normal file
@@ -0,0 +1,118 @@
|
||||
prompt = {
|
||||
"intro": "",
|
||||
"examples": [],
|
||||
"template": """<html> {html} </html>
|
||||
|
||||
You are a helpful assistant that can assist with web navigation tasks.
|
||||
You are given a simplified html webpage and a task description.
|
||||
Your goal is to complete the task. You can use the provided functions below to interact with the current webpage.
|
||||
|
||||
#Provided functions:
|
||||
def click(element_id: str) -> None:
|
||||
\"\"\"
|
||||
Click on the element with the specified id.
|
||||
|
||||
Args:
|
||||
element_id: The id of the element.
|
||||
\"\"\"
|
||||
|
||||
def hover(element_id: str) -> None:
|
||||
\"\"\"
|
||||
Hover on the element with the specified id.
|
||||
|
||||
Args:
|
||||
element_id: The id of the element.
|
||||
\"\"\"
|
||||
|
||||
def select(element_id: str, option: str) -> None:
|
||||
\"\"\"
|
||||
Select an option from a dropdown.
|
||||
|
||||
Args:
|
||||
element_id: The id of the element.
|
||||
option: Value of the option to select.
|
||||
\"\"\"
|
||||
|
||||
def type_string(element_id: str, content: str, press_enter: bool) -> None:
|
||||
\"\"\"
|
||||
Type a string into the element with the specified id.
|
||||
|
||||
Args:
|
||||
element_id: The id of the element.
|
||||
content: The string to type.
|
||||
press_enter: Whether to press enter after typing the string.
|
||||
\"\"\"
|
||||
|
||||
def scroll_page(direction: Literal['up', 'down']) -> None:
|
||||
\"\"\"
|
||||
Scroll down/up one page.
|
||||
|
||||
Args:
|
||||
direction: The direction to scroll.
|
||||
\"\"\"
|
||||
|
||||
def go(direction: Literal['forward', 'backward']) -> None:
|
||||
\"\"\"
|
||||
Go forward/backward
|
||||
|
||||
Args:
|
||||
direction: The direction to go to.
|
||||
\"\"\"
|
||||
|
||||
def jump_to(url: str, new_tab: bool) -> None:
|
||||
\"\"\"
|
||||
Jump to the specified url.
|
||||
|
||||
Args:
|
||||
url: The url to jump to.
|
||||
new_tab: Whether to open the url in a new tab.
|
||||
\"\"\"
|
||||
|
||||
def switch_tab(tab_index: int) -> None:
|
||||
\"\"\"
|
||||
Switch to the specified tab.
|
||||
|
||||
Args:
|
||||
tab_index: The index of the tab to switch to.
|
||||
\"\"\"
|
||||
|
||||
def user_input(message: str) -> str:
|
||||
\"\"\"
|
||||
Wait for user input.
|
||||
|
||||
Args:
|
||||
message: The message to display to the user.
|
||||
|
||||
Returns: The user input.
|
||||
\"\"\"
|
||||
|
||||
def finish(answer: Optional[str]) -> None:
|
||||
\"\"\"
|
||||
Finish the task (optionally with an answer).
|
||||
|
||||
Args:
|
||||
answer: The answer to the task.
|
||||
\"\"\"
|
||||
|
||||
#Previous commands: {previous_action}
|
||||
|
||||
#Window tabs: {tabs}
|
||||
|
||||
#Current viewport (pages): {position}
|
||||
|
||||
#Task: {objective}
|
||||
|
||||
You should output one command to interact to the currrent webpage.
|
||||
You should add a brief comment to your command to explain your reasoning and thinking process.
|
||||
""",
|
||||
"finale": "",
|
||||
"meta_data": {
|
||||
"observation": "html",
|
||||
"action_type": "id_html_nasc_tree",
|
||||
"keywords": ["url", "html", "objective", "position", "previous_action", "tabs"],
|
||||
"prompt_constructor": "NewASPromptConstructor",
|
||||
"answer_phrase": "",
|
||||
"action_splitter": "#"
|
||||
},
|
||||
}
|
||||
|
||||
82
webarena/agent/prompts/raw/p_cot_id_actree_2s.py
Normal file
82
webarena/agent/prompts/raw/p_cot_id_actree_2s.py
Normal file
@@ -0,0 +1,82 @@
|
||||
prompt = {
|
||||
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
|
||||
|
||||
Here's the information you'll have:
|
||||
The user's objective: This is the task you're trying to complete.
|
||||
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
|
||||
The current web page's URL: This is the page you're currently navigating.
|
||||
The open tabs: These are the tabs you have open.
|
||||
The previous action: This is the action you just performed. It may be helpful to track your progress.
|
||||
|
||||
The actions you can perform fall into several categories:
|
||||
|
||||
Page Operation Actions:
|
||||
`click [id]`: This action clicks on an element with a specific id on the webpage.
|
||||
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
|
||||
`hover [id]`: Hover over an element with id.
|
||||
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
|
||||
`scroll [direction=down|up]`: Scroll the page up or down.
|
||||
|
||||
Tab Management Actions:
|
||||
`new_tab`: Open a new, empty browser tab.
|
||||
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
|
||||
`close_tab`: Close the currently active tab.
|
||||
|
||||
URL Navigation Actions:
|
||||
`goto [url]`: Navigate to a specific URL.
|
||||
`go_back`: Navigate to the previously viewed page.
|
||||
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
|
||||
|
||||
Completion Action:
|
||||
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
|
||||
|
||||
Homepage:
|
||||
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
|
||||
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
|
||||
|
||||
To be successful, it is very important to follow the following rules:
|
||||
1. You should only issue an action that is valid given the current observation
|
||||
2. You should only issue one action at a time.
|
||||
3. You should follow the examples to reason step by step and then issue the next action.
|
||||
4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
|
||||
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
|
||||
"examples": [
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
|
||||
[1749] StaticText '$279.49'
|
||||
[1757] button 'Add to Cart'
|
||||
[1760] button 'Add to Wish List'
|
||||
[1761] button 'Add to Compare'
|
||||
URL: http://onestopmarket.com/office-products/office-electronics.html
|
||||
OBJECTIVE: What is the price of HP Inkjet Fax Machine
|
||||
PREVIOUS ACTION: None""",
|
||||
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
|
||||
),
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[164] textbox 'Search' focused: True required: False
|
||||
[171] button 'Go'
|
||||
[174] link 'Find directions between two points'
|
||||
[212] heading 'Search Results'
|
||||
[216] button 'Close'
|
||||
URL: http://openstreetmap.org
|
||||
OBJECTIVE: Show me the restaurants near CMU
|
||||
PREVIOUS ACTION: None""",
|
||||
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
|
||||
),
|
||||
],
|
||||
"template": """OBSERVATION:
|
||||
{observation}
|
||||
URL: {url}
|
||||
OBJECTIVE: {objective}
|
||||
PREVIOUS ACTION: {previous_action}""",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": ["url", "objective", "observation", "previous_action"],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
},
|
||||
}
|
||||
82
webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py
Normal file
82
webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py
Normal file
@@ -0,0 +1,82 @@
|
||||
prompt = {
|
||||
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
|
||||
|
||||
Here's the information you'll have:
|
||||
The user's objective: This is the task you're trying to complete.
|
||||
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
|
||||
The current web page's URL: This is the page you're currently navigating.
|
||||
The open tabs: These are the tabs you have open.
|
||||
The previous action: This is the action you just performed. It may be helpful to track your progress.
|
||||
|
||||
The actions you can perform fall into several categories:
|
||||
|
||||
Page Operation Actions:
|
||||
`click [id]`: This action clicks on an element with a specific id on the webpage.
|
||||
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
|
||||
`hover [id]`: Hover over an element with id.
|
||||
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
|
||||
`scroll [direction=down|up]`: Scroll the page up or down.
|
||||
|
||||
Tab Management Actions:
|
||||
`new_tab`: Open a new, empty browser tab.
|
||||
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
|
||||
`close_tab`: Close the currently active tab.
|
||||
|
||||
URL Navigation Actions:
|
||||
`goto [url]`: Navigate to a specific URL.
|
||||
`go_back`: Navigate to the previously viewed page.
|
||||
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
|
||||
|
||||
Completion Action:
|
||||
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
|
||||
|
||||
Homepage:
|
||||
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
|
||||
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
|
||||
|
||||
To be successful, it is very important to follow the following rules:
|
||||
1. You should only issue an action that is valid given the current observation
|
||||
2. You should only issue one action at a time.
|
||||
3. You should follow the examples to reason step by step and then issue the next action.
|
||||
4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
|
||||
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
|
||||
"examples": [
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
|
||||
[1749] StaticText '$279.49'
|
||||
[1757] button 'Add to Cart'
|
||||
[1760] button 'Add to Wish List'
|
||||
[1761] button 'Add to Compare'
|
||||
URL: http://onestopmarket.com/office-products/office-electronics.html
|
||||
OBJECTIVE: What is the price of HP Inkjet Fax Machine
|
||||
PREVIOUS ACTION: None""",
|
||||
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
|
||||
),
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[164] textbox 'Search' focused: True required: False
|
||||
[171] button 'Go'
|
||||
[174] link 'Find directions between two points'
|
||||
[212] heading 'Search Results'
|
||||
[216] button 'Close'
|
||||
URL: http://openstreetmap.org
|
||||
OBJECTIVE: Show me the restaurants near CMU
|
||||
PREVIOUS ACTION: None""",
|
||||
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
|
||||
),
|
||||
],
|
||||
"template": """OBSERVATION:
|
||||
{observation}
|
||||
URL: {url}
|
||||
OBJECTIVE: {objective}
|
||||
PREVIOUS ACTION: {previous_action}""",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": ["url", "objective", "observation", "previous_action"],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
},
|
||||
}
|
||||
80
webarena/agent/prompts/raw/p_direct_id_actree_2s.py
Normal file
80
webarena/agent/prompts/raw/p_direct_id_actree_2s.py
Normal file
@@ -0,0 +1,80 @@
|
||||
prompt = {
|
||||
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
|
||||
|
||||
Here's the information you'll have:
|
||||
The user's objective: This is the task you're trying to complete.
|
||||
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
|
||||
The current web page's URL: This is the page you're currently navigating.
|
||||
The open tabs: These are the tabs you have open.
|
||||
The previous action: This is the action you just performed. It may be helpful to track your progress.
|
||||
|
||||
The actions you can perform fall into several categories:
|
||||
|
||||
Page Operation Actions:
|
||||
`click [id]`: This action clicks on an element with a specific id on the webpage.
|
||||
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
|
||||
`hover [id]`: Hover over an element with id.
|
||||
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
|
||||
`scroll [direction=down|up]`: Scroll the page up or down.
|
||||
|
||||
Tab Management Actions:
|
||||
`new_tab`: Open a new, empty browser tab.
|
||||
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
|
||||
`close_tab`: Close the currently active tab.
|
||||
|
||||
URL Navigation Actions:
|
||||
`goto [url]`: Navigate to a specific URL.
|
||||
`go_back`: Navigate to the previously viewed page.
|
||||
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
|
||||
|
||||
Completion Action:
|
||||
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
|
||||
|
||||
Homepage:
|
||||
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
|
||||
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
|
||||
|
||||
To be successful, it is very important to follow the following rules:
|
||||
1. You should only issue an action that is valid given the current observation
|
||||
2. You should only issue one action at a time.
|
||||
3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.
|
||||
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
|
||||
"examples": [
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
|
||||
[1749] StaticText '$279.49'
|
||||
[1757] button 'Add to Cart'
|
||||
[1760] button 'Add to Wish List'
|
||||
[1761] button 'Add to Compare'
|
||||
URL: http://onestopmarket.com/office-products/office-electronics.html
|
||||
OBJECTIVE: What is the price of HP Inkjet Fax Machine
|
||||
PREVIOUS ACTION: None""",
|
||||
"```stop [$279.49]```",
|
||||
),
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[164] textbox 'Search' focused: True required: False
|
||||
[171] button 'Go'
|
||||
[174] link 'Find directions between two points'
|
||||
[212] heading 'Search Results'
|
||||
[216] button 'Close'
|
||||
URL: http://openstreetmap.org
|
||||
OBJECTIVE: Show me the restaurants near CMU
|
||||
PREVIOUS ACTION: None""",
|
||||
"```type [164] [restaurants near CMU] [1]```",
|
||||
),
|
||||
],
|
||||
"template": """OBSERVATION:
|
||||
{observation}
|
||||
URL: {url}
|
||||
OBJECTIVE: {objective}
|
||||
PREVIOUS ACTION: {previous_action}""",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": ["url", "objective", "observation", "previous_action"],
|
||||
"prompt_constructor": "DirectPromptConstructor",
|
||||
"action_splitter": "```"
|
||||
},
|
||||
}
|
||||
81
webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py
Normal file
81
webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py
Normal file
@@ -0,0 +1,81 @@
|
||||
prompt = {
|
||||
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
|
||||
|
||||
Here's the information you'll have:
|
||||
The user's objective: This is the task you're trying to complete.
|
||||
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
|
||||
The current web page's URL: This is the page you're currently navigating.
|
||||
The open tabs: These are the tabs you have open.
|
||||
The previous action: This is the action you just performed. It may be helpful to track your progress.
|
||||
|
||||
The actions you can perform fall into several categories:
|
||||
|
||||
Page Operation Actions:
|
||||
`click [id]`: This action clicks on an element with a specific id on the webpage.
|
||||
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
|
||||
`hover [id]`: Hover over an element with id.
|
||||
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
|
||||
`scroll [direction=down|up]`: Scroll the page up or down.
|
||||
|
||||
Tab Management Actions:
|
||||
`new_tab`: Open a new, empty browser tab.
|
||||
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
|
||||
`close_tab`: Close the currently active tab.
|
||||
|
||||
URL Navigation Actions:
|
||||
`goto [url]`: Navigate to a specific URL.
|
||||
`go_back`: Navigate to the previously viewed page.
|
||||
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
|
||||
|
||||
Completion Action:
|
||||
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
|
||||
|
||||
Homepage:
|
||||
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
|
||||
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
|
||||
|
||||
To be successful, it is very important to follow the following rules:
|
||||
1. You should only issue an action that is valid given the current observation
|
||||
2. You should only issue one action at a time.
|
||||
4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```".
|
||||
5. Issue stop action when you think you have achieved the objective.""",
|
||||
"examples": [
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
|
||||
[1749] StaticText '$279.49'
|
||||
[1757] button 'Add to Cart'
|
||||
[1760] button 'Add to Wish List'
|
||||
[1761] button 'Add to Compare'
|
||||
URL: http://onestopmarket.com/office-products/office-electronics.html
|
||||
OBJECTIVE: What is the price of HP Inkjet Fax Machine
|
||||
PREVIOUS ACTION: None""",
|
||||
"```stop [$279.49]```",
|
||||
),
|
||||
(
|
||||
"""OBSERVATION:
|
||||
[164] textbox 'Search' focused: True required: False
|
||||
[171] button 'Go'
|
||||
[174] link 'Find directions between two points'
|
||||
[212] heading 'Search Results'
|
||||
[216] button 'Close'
|
||||
URL: http://openstreetmap.org
|
||||
OBJECTIVE: Show me the restaurants near CMU
|
||||
PREVIOUS ACTION: None""",
|
||||
"```type [164] [restaurants near CMU] [1]```",
|
||||
),
|
||||
],
|
||||
"template": """OBSERVATION:
|
||||
{observation}
|
||||
URL: {url}
|
||||
OBJECTIVE: {objective}
|
||||
PREVIOUS ACTION: {previous_action}""",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": ["url", "objective", "observation", "previous_action"],
|
||||
"prompt_constructor": "CoTPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```"
|
||||
},
|
||||
}
|
||||
83
webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py
Normal file
83
webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py
Normal file
@@ -0,0 +1,83 @@
|
||||
prompt = {
|
||||
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:
|
||||
|
||||
Page Operation Actions:
|
||||
`click [id]`: This action clicks on an element with a specific id on the webpage.
|
||||
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
|
||||
`hover [id]`: Hover over an element with id.
|
||||
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
|
||||
`scroll [direction=down|up]`: Scroll the page up or down.
|
||||
|
||||
Tab Management Actions:
|
||||
`new_tab`: Open a new, empty browser tab.
|
||||
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
|
||||
`close_tab`: Close the currently active tab.
|
||||
|
||||
URL Navigation Actions:
|
||||
`goto [url]`: Navigate to a specific URL.
|
||||
`go_back`: Navigate to the previously viewed page.
|
||||
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
|
||||
|
||||
Completion Action:
|
||||
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
|
||||
|
||||
Homepage:
|
||||
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
|
||||
|
||||
You can only issue one action at a time""",
|
||||
|
||||
"examples": [
|
||||
(
|
||||
"""Observation:
|
||||
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
|
||||
[1749] StaticText '$279.49'
|
||||
[1757] button 'Add to Cart'
|
||||
[1760] button 'Add to Wish List'
|
||||
[1761] button 'Add to Compare'
|
||||
URL: http://onestopmarket.com/office-products/office-electronics.html
|
||||
Objective: What is the price of HP Inkjet Fax Machine
|
||||
Previous action: None""",
|
||||
"```stop [$279.49]```",
|
||||
),
|
||||
(
|
||||
"""Observation:
|
||||
[164] textbox 'Search' focused: True required: False
|
||||
[171] button 'Go'
|
||||
[174] link 'Find directions between two points'
|
||||
[212] heading 'Search Results'
|
||||
[216] button 'Close'
|
||||
URL: http://openstreetmap.org
|
||||
Objective: Show me the restaurants near CMU
|
||||
Previous action: None""",
|
||||
"```type [164] [restaurants near CMU] [1]```",
|
||||
),
|
||||
(
|
||||
"""Observation:
|
||||
[2036] button 'Sort by: New' hasPopup: menu expanded: False
|
||||
[587] link 'US Marine’s adoption of Afghan war orphan voided'
|
||||
[989] time 'March 30, 2023 at 15:03:48 AM UTC'
|
||||
[602] link 'York student uses AI chatbot to get parking fine revoked'
|
||||
[1025] time 'March 15, 2023 at 7:48:34 AM UTC'
|
||||
[617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'
|
||||
[1025] time 'March 2, 2023 at 3:46:01 AM UTC'
|
||||
URL: http://reddit.com/f/news/new
|
||||
Objective: Open the most recent post that was published prior to March 1st.
|
||||
Previous action: None""",
|
||||
"```scroll [down]```",
|
||||
)
|
||||
],
|
||||
"template": """Observation:
|
||||
{observation}
|
||||
URL: {url}
|
||||
Objective: {objective}
|
||||
Previous action: {previous_action}""",
|
||||
"meta_data": {
|
||||
"observation": "accessibility_tree",
|
||||
"action_type": "id_accessibility_tree",
|
||||
"keywords": ["url", "objective", "observation", "previous_action"],
|
||||
"prompt_constructor": "DirectPromptConstructor",
|
||||
"answer_phrase": "In summary, the next action I will perform is",
|
||||
"action_splitter": "```",
|
||||
"force_prefix": "```"
|
||||
},
|
||||
}
|
||||
41
webarena/agent/prompts/raw/test_prompt.py
Normal file
41
webarena/agent/prompts/raw/test_prompt.py
Normal file
@@ -0,0 +1,41 @@
|
||||
prompt = {
|
||||
"intro": "",
|
||||
"examples": [],
|
||||
"template": """<html> {html} </html>
|
||||
|
||||
You are a helpful assistant that can assist with web navigation tasks.
|
||||
You are given a simplified html webpage and a task description.
|
||||
Your goal is to complete the task. You can perform the specified operations below to interact with the webpage.
|
||||
|
||||
#Valid operations: - #Click# id: Click on the element with the specified id
|
||||
- #Scroll_up#: Scroll up 1 page.
|
||||
- #Scroll_down#: Scroll down 1 page.
|
||||
- #Go_backward#: Go back to the previous page.
|
||||
- #Go_forward#: Go forward to the next page.
|
||||
- #Hover# id: Hover over the element with the specified id.
|
||||
- #Type# id "text": Type in the text at the element with the specified id.
|
||||
- #Select# id "option": Select the option at the element with the specified id.
|
||||
- #Record# "content": Mark content that is useful in answering the question.
|
||||
- #Answer# "text": output the text as the answer to the user.
|
||||
- #Exit#: Complete the task and exit the program.
|
||||
|
||||
#Current viewport position: {position}
|
||||
|
||||
#Previous Operation: {previous_action}
|
||||
|
||||
#Task: {objective}
|
||||
""",
|
||||
"finale": """
|
||||
Your output SHOULD be in the following format:
|
||||
#Operation: {Next operation to perform}
|
||||
""",
|
||||
"meta_data": {
|
||||
"observation": "html",
|
||||
"action_type": "id_html_tree",
|
||||
"keywords": ["url", "html", "objective", "position", "previous_action"],
|
||||
"prompt_constructor": "MyPromptConstructor",
|
||||
"answer_phrase": "",
|
||||
"action_splitter": "#"
|
||||
},
|
||||
}
|
||||
|
||||
26
webarena/agent/prompts/to_json.py
Normal file
26
webarena/agent/prompts/to_json.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import glob
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
# use the current directory as the root
|
||||
def run() -> None:
|
||||
"""Convert all python files in agent/prompts to json files in agent/prompts/jsons
|
||||
|
||||
Python files are easiser to edit
|
||||
"""
|
||||
for p_file in glob.glob(f"agent/prompts/raw/*.py"):
|
||||
# import the file as a module
|
||||
base_name = os.path.basename(p_file).replace(".py", "")
|
||||
module = importlib.import_module(f"agent.prompts.raw.{base_name}")
|
||||
prompt = module.prompt
|
||||
# save the prompt as a json file
|
||||
os.makedirs("agent/prompts/jsons", exist_ok=True)
|
||||
with open(f"agent/prompts/jsons/{base_name}.json", "w+") as f:
|
||||
json.dump(prompt, f, indent=2)
|
||||
print(f"Done convert python files to json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user